Reformat and don't pass back default parameters to URL

app/local_settings.example.py

@@ -5,98 +5,64 @@ OPENSEARCH_TLS = True
 OPENSEARCH_USERNAME = "admin"
 OPENSEARCH_PASSWORD = ""
 
-OPENSEARCH_INDEX_MAIN = "main"
-OPENSEARCH_INDEX_META = "meta"
-OPENSEARCH_INDEX_INT = "int"
+OPENSEARCH_INDEX_MAIN = "pathogen-main"
+OPENSEARCH_INDEX_META = "pathogen-meta"
+OPENSEARCH_INDEX_INT = "pathogen-int"
 
 OPENSEARCH_MAIN_SIZES = ["20", "50", "100", "200", "400", "800"]
 OPENSEARCH_MAIN_SIZES_ANON = ["20", "50", "100"]
-OPENSEARCH_MAIN_WILDCARD = True
-OPENSEARCH_MAIN_WILDCARD_ANON = False
-OPENSEARCH_MAIN_SOURCES = ["irc", "dis", "all"]
+OPENSEARCH_MAIN_SOURCES = ["dis", "4ch", "all"]
+OPENSEARCH_SOURCES_RESTRICTED = ["irc"]
+
+# Manticore settings
+MANTICORE_URL = "http://monolith-db-1:9308"
+MANTICORE_INDEX_MAIN = "main"
+MANTICORE_INDEX_META = "meta"
+MANTICORE_INDEX_INT = "int"
+
+MANTICORE_MAIN_SIZES = ["20", "50", "100", "200", "400", "800"]
+MANTICORE_MAIN_SIZES_ANON = ["20", "50", "100"]
+MANTICORE_MAIN_SOURCES = ["dis", "4ch", "all"]
+MANTICORE_SOURCES_RESTRICTED = ["irc"]
+
 DRILLDOWN_RESULTS_PER_PAGE = 15
+DRILLDOWN_DEFAULT_PARAMS = {
+    "size": "20",
+    "index": "main",
+    "sorting": "desc",
+    "source": "4ch",
+}
 
 # Encryption
-ENCRYPTION = False
-ENCRYPTION_KEY = b""
+# ENCRYPTION = False
+# ENCRYPTION_KEY = b""
 
 # Hashing
-HASHING = True
-HASHING_KEY = "xxx"
+# HASHING = True
+# HASHING_KEY = "xxx"
 
 # Obfuscation
-OBFUSCATION = True
-# Fields obfuscate based on separators
-OBFUSCATE_FIELDS_SEP = ["date", "time"]
-# Fields to obfuscate based on length
-OBFUSCATE_FIELDS = ["ts"]
-OBFUSCATE_KEEP_RATIO = 0.9
-# DON'T obfuscate the last X fields of values separates by dashes
-OBFUSCATE_DASH_NUM = 2
-# DON'T obfuscate the last X fields of values separates by colons
-OBFUSCATE_COLON_NUM = 1
-
-SEARCH_FIELDS_DENY = ["ts", "date", "time"]
-
-DELAY_RESULTS = True
-# Delay results by this many days
-DELAY_DURATION = 10
-
-# Common to encryption and hashing
-WHITELIST_FIELDS = [
-    "ts",
-    "date",
-    "time",
-    "sentiment",
-    "version_sentiment",
-    "tokens",
-    "num_chans",
-    "num_users",
-    "online",
-    "src",
-    "exemption",
-    "hidden",
-    "type",
-]
-
-# Don't obfuscate these parameters, or lookup hashes in them
-NO_OBFUSCATE_PARAMS = [
-    "query",
-    #    "query_full",
-    "size",
-    "source",
-    "sorting",
-    "tags",
-    "index",
-    "dedup",
-    "check_sentiment",
-    "sentiment_method",
-    "dates",
-    "sort",
-    "page",
-]
-
-# Don't allow tag search for these parameters
-TAG_SEARCH_DENY = [
-    "query",
-    "query_full",
-    "size",
-    "source",
-    "sorting",
-    "tags",
-    "index",
-    "dedup",
-    "check_sentiment",
-    "sentiment_method",
-    "dates",
-    "sort",
-    "page",
-]
-
+# OBFUSCATION = True
+# # Fields obfuscate based on separators
+# OBFUSCATE_FIELDS_SEP = ["date", "time"]
+# # Fields to obfuscate based on length
+# OBFUSCATE_FIELDS = ["ts"]
+# OBFUSCATE_KEEP_RATIO = 0.9
+# # DON'T obfuscate the last X fields of values separates by dashes
+# OBFUSCATE_DASH_NUM = 2
+# # DON'T obfuscate the last X fields of values separates by colons
+# OBFUSCATE_COLON_NUM = 1
+
+# SEARCH_FIELDS_DENY = ["ts", "date", "time"]
+
+# DELAY_RESULTS = True
+# # Delay results by this many days
+# DELAY_DURATION = 10
 
 OPENSEARCH_BLACKLISTED = {}
 
-# URLs
+
+# URLs\
 DOMAIN = "example.com"
 URL = f"https://{DOMAIN}"
 
@@ -121,8 +87,8 @@ SECRET_KEY = "a"
 STRIPE_ADMIN_COUPON = ""
 
 # Threshold
-THRESHOLD_ENDPOINT = ""
-THRESHOLD_API_KEY = "api_1"
+THRESHOLD_ENDPOINT = "http://threshold-app-1:13869"
+THRESHOLD_API_KEY = ""
 THRESHOLD_API_TOKEN = ""
 THRESHOLD_API_COUNTER = ""
 
@@ -138,3 +104,12 @@ META_QUERY_SIZE = 10000
 
 DEBUG = True
 PROFILER = False
+
+if DEBUG:
+    import socket  # only if you haven't already imported this
+
+    hostname, _, ips = socket.gethostbyname_ex(socket.gethostname())
+    INTERNAL_IPS = [ip[: ip.rfind(".")] + ".1" for ip in ips] + [
+        "127.0.0.1",
+        "10.0.2.2",
+    ]

app/settings.py

@@ -42,7 +42,6 @@ INSTALLED_APPS = [
     "crispy_bulma",
     "django_tables2",
     "django_tables2_bulma_template",
-
 ]
 CRISPY_TEMPLATE_PACK = "bulma"
 CRISPY_ALLOWED_TEMPLATE_PACKS = ("bulma",)
@@ -149,21 +148,21 @@ INTERNAL_IPS = [
 ]
 
 DEBUG_TOOLBAR_PANELS = [
-    'template_profiler_panel.panels.template.TemplateProfilerPanel',
-    'debug_toolbar.panels.history.HistoryPanel',
-    'debug_toolbar.panels.versions.VersionsPanel',
-    'debug_toolbar.panels.timer.TimerPanel',
-    'debug_toolbar.panels.settings.SettingsPanel',
-    'debug_toolbar.panels.headers.HeadersPanel',
-    'debug_toolbar.panels.request.RequestPanel',
-    'debug_toolbar.panels.sql.SQLPanel',
-    'debug_toolbar.panels.staticfiles.StaticFilesPanel',
-    'debug_toolbar.panels.templates.TemplatesPanel',
-    'debug_toolbar.panels.cache.CachePanel',
-    'debug_toolbar.panels.signals.SignalsPanel',
-    'debug_toolbar.panels.logging.LoggingPanel',
-    'debug_toolbar.panels.redirects.RedirectsPanel',
-    'debug_toolbar.panels.profiling.ProfilingPanel',
+    "template_profiler_panel.panels.template.TemplateProfilerPanel",
+    "debug_toolbar.panels.history.HistoryPanel",
+    "debug_toolbar.panels.versions.VersionsPanel",
+    "debug_toolbar.panels.timer.TimerPanel",
+    "debug_toolbar.panels.settings.SettingsPanel",
+    "debug_toolbar.panels.headers.HeadersPanel",
+    "debug_toolbar.panels.request.RequestPanel",
+    "debug_toolbar.panels.sql.SQLPanel",
+    "debug_toolbar.panels.staticfiles.StaticFilesPanel",
+    "debug_toolbar.panels.templates.TemplatesPanel",
+    "debug_toolbar.panels.cache.CachePanel",
+    "debug_toolbar.panels.signals.SignalsPanel",
+    "debug_toolbar.panels.logging.LoggingPanel",
+    "debug_toolbar.panels.redirects.RedirectsPanel",
+    "debug_toolbar.panels.profiling.ProfilingPanel",
 ]
 
 from app.local_settings import *  # noqa

app/urls.py

@@ -73,7 +73,7 @@ from core.views.ui.insights import (
 )
 
 urlpatterns = [
-    path('__debug__/', include('debug_toolbar.urls')),
+    path("__debug__/", include("debug_toolbar.urls")),
     path("", DrilldownTableView.as_view(), name="home"),
     path("search/", DrilldownTableView.as_view(), name="search"),
     path("about/", About.as_view(), name="about"),

core/lib/manticore.py

@@ -5,7 +5,7 @@ import manticoresearch
 from django.conf import settings
 
 from core.lib.processing import annotate_results, filter_blacklisted, parse_results
-from core.views.helpers import dedup_list
+from core.views import helpers
 
 
 def initialise_manticore():
@@ -68,7 +68,9 @@ def query_results(
     sort = None
     query_created = False
     source = None
-
+    print("BEFORE ADD DEFAULTS", query_params)
+    helpers.add_defaults(query_params)
+    print("AFTER ADD DEFAULTS", query_params)
     # Check size
     if request.user.is_anonymous:
         sizes = settings.MANTICORE_MAIN_SIZES_ANON
@@ -283,7 +285,7 @@ def query_results(
     if dedup:
         if not dedup_fields:
             dedup_fields = ["msg", "nick", "ident", "host", "net", "channel"]
-        results_parsed = dedup_list(results_parsed, dedup_fields)
+        results_parsed = helpers.dedup_list(results_parsed, dedup_fields)
     context = {
         "object_list": results_parsed,
         "card": results["hits"]["total"],

core/static/modal.js

@@ -8,7 +8,7 @@ var disableModal = function() {
     var modal_refresh = document.getElementsByClassName("modal-refresh");
     for(var i = 0; i < modal_refresh.length; i++) {
         modal_refresh[i].remove();
-}
+    }
 }
 
 var elements = document.querySelectorAll('.modal-background');

core/static/tabs.js

@@ -10,7 +10,7 @@ function initTabs(unique) {
             updateActiveContent(CONTENT, ACTIVE_CLASS, selected);
         })
     })
-  }
+}
 
 function updateActiveTab(TABS, ACTIVE_CLASS, selected) {
     TABS.forEach((tab) => {

core/templates/wm/panel.html

@@ -1,5 +1,5 @@
 
-  <nav class="panel">
+<nav class="panel">
     <p class="panel-heading" style="padding: .2em; line-height: .5em;">
         <i class="fa-solid fa-arrows-up-down-left-right has-text-grey-light"></i>
         {% block close_button %}
@@ -16,4 +16,4 @@
             {% endblock %}
         </div>
     </article>
-  </nav>
+</nav>

core/templates/wm/widget.html

@@ -35,4 +35,3 @@
     {% block custom_script_end %}
     {% endblock %}
 </script>
-  

core/views/helpers.py

@@ -9,6 +9,7 @@
 # from sortedcontainers import SortedSet
 
 # from core import r
+from django.conf import settings
 
 
 class SearchDenied:
@@ -23,6 +24,19 @@ class LookupDenied:
         self.value = value
 
 
+def remove_defaults(query_params):
+    for field, value in list(query_params.items()):
+        if field in settings.DRILLDOWN_DEFAULT_PARAMS:
+            if value == settings.DRILLDOWN_DEFAULT_PARAMS[field]:
+                del query_params[field]
+
+
+def add_defaults(query_params):
+    for field, value in settings.DRILLDOWN_DEFAULT_PARAMS.items():
+        if field not in query_params:
+            query_params[field] = value
+
+
 def dedup_list(data, check_keys):
     """
     Remove duplicate dictionaries from list.

core/views/ui/drilldown.py

@@ -21,6 +21,7 @@ from core.lib.threshold import (
     get_chans,
     get_users,
 )
+from core.views import helpers
 from core.views.ui.tables import DrilldownTable
 
 # from copy import deepcopy
@@ -155,11 +156,6 @@ def drilldown_search(request, return_context=False, template=None):
     # else:
     #     context = {"object_list": []}
 
-    # Remove null values
-    if "query_full" in query_params:
-        if query_params["query_full"] == "":
-            del query_params["query_full"]
-
     if "tags" in query_params:
         if query_params["tags"] == "":
             del query_params["tags"]
@@ -172,7 +168,9 @@ def drilldown_search(request, return_context=False, template=None):
 
     # Valid sizes
     context["sizes"] = sizes
-
+    print("BEFORE REMOVE DEFAULT", query_params)
+    helpers.remove_defaults(query_params)
+    print("AFTER REMOVE DEFAULT", query_params)
     url_params = urllib.parse.urlencode(query_params)
     context["client_uri"] = url_params
     context["params"] = query_params
@@ -199,17 +197,6 @@ def drilldown_search(request, return_context=False, template=None):
     clean_url_params = urllib.parse.urlencode(clean_params)
     context["uri"] = clean_url_params
 
-    # Warn users trying to use query string that the simple query supersedes it
-    if all([x in query_params for x in ["query", "query_full"]]):
-        context["message"] = (
-            "You are searching with both query types. "
-            "The simple query will be used. "
-            "The full query will be ignored. "
-            "Remove the text from the simple query if you wish "
-            "to use the full query."
-        )
-        context["class"] = "warning"
-
     # unique = str(uuid.uuid4())[:8]
     if return_context:
         return context

docker/Dockerfile

@@ -15,4 +15,4 @@ WORKDIR /code
 COPY requirements.dev.txt /code/
 RUN python -m venv /venv
 RUN . /venv/bin/activate && pip install -r requirements.dev.txt
-CMD . /venv/bin/activate && exec python manage.py runserver --nothreading 0.0.0.0:8000
+CMD . /venv/bin/activate && exec python manage.py runserver 0.0.0.0:8000