Update to run with Podman

app/local_settings.example.py

@@ -1,3 +1,5 @@
+from os import getenv
+
 # Elasticsearch settings
 ELASTICSEARCH_URL = "10.1.0.1"
 ELASTICSEARCH_PORT = 9200
@@ -104,3 +106,8 @@ META_QUERY_SIZE = 10000
 
 DEBUG = True
 PROFILER = False
+
+REDIS_HOST = getenv("REDIS_HOST", "redis_fisk_dev")
+REDIS_PASSWORD = getenv("REDIS_PASSWORD", "changeme")
+REDIS_DB = int(getenv("REDIS_DB", "10"))
+REDIS_PORT = int(getenv("REDIS_PORT", "6379"))

app/local_settings.py

@@ -0,0 +1,87 @@
+from os import getenv
+
+trues = ("t", "true", "yes", "y", "1")
+
+# Elasticsearch settings
+ELASTICSEARCH_URL = getenv("ELASTICSEARCH_URL", "10.1.0.1")
+ELASTICSEARCH_PORT = int(getenv("ELASTICSEARCH_PORT", "9200"))
+ELASTICSEARCH_TLS = getenv("ELASTICSEARCH_TLS", "True").lower() in trues
+ELASTICSEARCH_USERNAME = getenv("ELASTICSEARCH_USERNAME", "admin")
+ELASTICSEARCH_PASSWORD = getenv("ELASTICSEARCH_PASSWORD", "secret")
+
+# Manticore settings
+MANTICORE_URL = getenv("MANTICORE_URL", "http://example-db-1:9308")
+
+DB_BACKEND = getenv("DB_BACKEND", "MANTICORE")
+
+# Common DB settings
+INDEX_MAIN = getenv("INDEX_MAIN", "main")
+INDEX_RESTRICTED = getenv("INDEX_RESTRICTED", "restricted")
+INDEX_META = getenv("INDEX_META", "meta")
+INDEX_INT = getenv("INDEX_INT", "internal")
+INDEX_RULE_STORAGE = getenv("INDEX_RULE_STORAGE", "rule_storage")
+
+MAIN_SIZES = getenv("MAIN_SIZES", "1,5,15,30,50,100,250,500,1000").split(",")
+MAIN_SIZES_ANON = getenv("MAIN_SIZES_ANON", "1,5,15,30,50,100").split(",")
+MAIN_SOURCES = getenv("MAIN_SOURCES", "dis,4ch,all").split(",")
+SOURCES_RESTRICTED = getenv("SOURCES_RESTRICTED", "irc").split(",")
+CACHE = getenv("CACHE", "False").lower() in trues
+CACHE_TIMEOUT = int(getenv("CACHE_TIMEOUT", "2"))
+
+DRILLDOWN_RESULTS_PER_PAGE = int(getenv("DRILLDOWN_RESULTS_PER_PAGE", "15"))
+DRILLDOWN_DEFAULT_PARAMS = {
+    "size": getenv("DRILLDOWN_DEFAULT_SIZE", "15"),
+    "index": getenv("DRILLDOWN_DEFAULT_INDEX", "main"),
+    "sorting": getenv("DRILLDOWN_DEFAULT_SORTING", "desc"),
+    "source": getenv("DRILLDOWN_DEFAULT_SOURCE", "all"),
+}
+
+# URLs
+DOMAIN = getenv("DOMAIN", "example.com")
+URL = getenv("URL", f"https://{DOMAIN}")
+
+# Access control
+ALLOWED_HOSTS = getenv("ALLOWED_HOSTS", f"127.0.0.1,{DOMAIN}").split(",")
+
+# CSRF
+CSRF_TRUSTED_ORIGINS = getenv("CSRF_TRUSTED_ORIGINS", URL).split(",")
+
+# Stripe
+BILLING_ENABLED = getenv("BILLING_ENABLED", "false").lower() in trues
+STRIPE_TEST = getenv("STRIPE_TEST", "True").lower() in trues
+STRIPE_API_KEY_TEST = getenv("STRIPE_API_KEY_TEST", "")
+STRIPE_PUBLIC_API_KEY_TEST = getenv("STRIPE_PUBLIC_API_KEY_TEST", "")
+STRIPE_API_KEY_PROD = getenv("STRIPE_API_KEY_PROD", "")
+STRIPE_PUBLIC_API_KEY_PROD = getenv("STRIPE_PUBLIC_API_KEY_PROD", "")
+STRIPE_ENDPOINT_SECRET = getenv("STRIPE_ENDPOINT_SECRET", "")
+STATIC_ROOT = getenv("STATIC_ROOT", "")
+SECRET_KEY = getenv("SECRET_KEY", "a")
+STRIPE_ADMIN_COUPON = getenv("STRIPE_ADMIN_COUPON", "")
+
+# Threshold
+THRESHOLD_ENDPOINT = getenv("THRESHOLD_ENDPOINT", "http://threshold:13869")
+THRESHOLD_API_KEY = getenv("THRESHOLD_API_KEY", "api_1")
+THRESHOLD_API_TOKEN = getenv("THRESHOLD_API_TOKEN", "")
+THRESHOLD_API_COUNTER = getenv("THRESHOLD_API_COUNTER", "")
+
+# NickTrace
+NICKTRACE_MAX_ITERATIONS = int(getenv("NICKTRACE_MAX_ITERATIONS", "4"))
+NICKTRACE_MAX_CHUNK_SIZE = int(getenv("NICKTRACE_MAX_CHUNK_SIZE", "500"))
+NICKTRACE_QUERY_SIZE = int(getenv("NICKTRACE_QUERY_SIZE", "10000"))
+
+# Meta
+META_MAX_ITERATIONS = int(getenv("META_MAX_ITERATIONS", "4"))
+META_MAX_CHUNK_SIZE = int(getenv("META_MAX_CHUNK_SIZE", "500"))
+META_QUERY_SIZE = int(getenv("META_QUERY_SIZE", "10000"))
+
+DEBUG = getenv("DEBUG", "True").lower() in trues
+PROFILER = getenv("PROFILER", "False").lower() in trues
+
+REDIS_HOST = getenv("REDIS_HOST", "redis_neptune_dev")
+REDIS_PASSWORD = getenv("REDIS_PASSWORD", "changeme")
+REDIS_DB = int(getenv("REDIS_DB", "1"))
+REDIS_DB_CACHE = int(getenv("REDIS_DB_CACHE", "10"))
+REDIS_PORT = int(getenv("REDIS_PORT", "6379"))
+
+# Elasticsearch blacklist
+ELASTICSEARCH_BLACKLISTED = {}

app/settings.py

@@ -47,19 +47,6 @@ INSTALLED_APPS = [
     "cachalot",
 ]
 
-# Performance optimisations
-CACHES = {
-    "default": {
-        "BACKEND": "django_redis.cache.RedisCache",
-        "LOCATION": "unix:///var/run/socks/redis.sock",
-        "OPTIONS": {
-            "db": "10",
-            # "parser_class": "django_redis.cache.RedisCache",
-            "pool_class": "redis.BlockingConnectionPool",
-        },
-    }
-}
-
 CRISPY_TEMPLATE_PACK = "bulma"
 CRISPY_ALLOWED_TEMPLATE_PACKS = ("bulma",)
 DJANGO_TABLES2_TEMPLATE = "django-tables2/bulma.html"
@@ -163,7 +150,7 @@ REST_FRAMEWORK = {
 
 INTERNAL_IPS = [
     "127.0.0.1",
-    "10.1.10.11",
+    # "10.1.10.11",
 ]
 
 DEBUG_TOOLBAR_PANELS = [
@@ -187,6 +174,21 @@ DEBUG_TOOLBAR_PANELS = [
 
 from app.local_settings import *  # noqa
 
+# Performance optimisations
+CACHES = {
+    "default": {
+        "BACKEND": "django_redis.cache.RedisCache",
+        # "LOCATION": "unix:///var/run/socks/redis.sock",
+        "LOCATION": f"redis://{REDIS_HOST}:{REDIS_PORT}",
+        "OPTIONS": {
+            "db": REDIS_DB_CACHE,
+            # "parser_class": "django_redis.cache.RedisCache",
+            "PASSWORD": REDIS_PASSWORD,
+            "pool_class": "redis.BlockingConnectionPool",
+        },
+    }
+}
+
 if PROFILER:  # noqa - trust me its there
     import pyroscope
 

app/urls.py

@@ -58,6 +58,9 @@ from core.views.manage.threshold.threshold import (
     ThresholdIRCOverview,
 )
 
+# Stats
+from core.views.manage.monolith import stats
+
 # Main tool pages
 from core.views.ui.drilldown import (  # DrilldownTableView,; Drilldown,
     DrilldownContextModal,
@@ -311,4 +314,14 @@ urlpatterns = [
         notifications.RuleClear.as_view(),
         name="rule_clear",
     ),
+    path(
+        "manage/monolith/stats/",
+        stats.MonolithStats.as_view(),
+        name="monolith_stats",
+    ),
+    path(
+        "manage/monolith/stats_db/<str:type>/",
+        stats.MonolithDBStats.as_view(),
+        name="monolith_stats_db",
+    )
 ] + static(settings.STATIC_URL, document_root=settings.STATIC_ROOT)