Reformat and don't pass back default parameters to URL

2022-09-05 07:20:30 +01:00 · 2022-09-05 07:20:30 +01:00 · 11dbe3e094
parent ba57c378cd
commit 11dbe3e094
64 changed files with 3843 additions and 3867 deletions
--- a/app/local_settings.example.py
+++ b/app/local_settings.example.py
@ -5,98 +5,64 @@ OPENSEARCH_TLS = True
 OPENSEARCH_USERNAME = "admin"
 OPENSEARCH_PASSWORD = ""

-OPENSEARCH_INDEX_MAIN = "main"
-OPENSEARCH_INDEX_META = "meta"
-OPENSEARCH_INDEX_INT = "int"
+OPENSEARCH_INDEX_MAIN = "pathogen-main"
+OPENSEARCH_INDEX_META = "pathogen-meta"
+OPENSEARCH_INDEX_INT = "pathogen-int"

 OPENSEARCH_MAIN_SIZES = ["20", "50", "100", "200", "400", "800"]
 OPENSEARCH_MAIN_SIZES_ANON = ["20", "50", "100"]
-OPENSEARCH_MAIN_WILDCARD = True
-OPENSEARCH_MAIN_WILDCARD_ANON = False
-OPENSEARCH_MAIN_SOURCES = ["irc", "dis", "all"]
+OPENSEARCH_MAIN_SOURCES = ["dis", "4ch", "all"]
+OPENSEARCH_SOURCES_RESTRICTED = ["irc"]
+
+# Manticore settings
+MANTICORE_URL = "http://monolith-db-1:9308"
+MANTICORE_INDEX_MAIN = "main"
+MANTICORE_INDEX_META = "meta"
+MANTICORE_INDEX_INT = "int"
+
+MANTICORE_MAIN_SIZES = ["20", "50", "100", "200", "400", "800"]
+MANTICORE_MAIN_SIZES_ANON = ["20", "50", "100"]
+MANTICORE_MAIN_SOURCES = ["dis", "4ch", "all"]
+MANTICORE_SOURCES_RESTRICTED = ["irc"]
+
 DRILLDOWN_RESULTS_PER_PAGE = 15
+DRILLDOWN_DEFAULT_PARAMS = {
+    "size": "20",
+    "index": "main",
+    "sorting": "desc",
+    "source": "4ch",
+}

 # Encryption
-ENCRYPTION = False
-ENCRYPTION_KEY = b""
+# ENCRYPTION = False
+# ENCRYPTION_KEY = b""

 # Hashing
-HASHING = True
-HASHING_KEY = "xxx"
+# HASHING = True
+# HASHING_KEY = "xxx"

 # Obfuscation
-OBFUSCATION = True
-# Fields obfuscate based on separators
-OBFUSCATE_FIELDS_SEP = ["date", "time"]
-# Fields to obfuscate based on length
-OBFUSCATE_FIELDS = ["ts"]
-OBFUSCATE_KEEP_RATIO = 0.9
-# DON'T obfuscate the last X fields of values separates by dashes
-OBFUSCATE_DASH_NUM = 2
-# DON'T obfuscate the last X fields of values separates by colons
-OBFUSCATE_COLON_NUM = 1
+# OBFUSCATION = True
+# # Fields obfuscate based on separators
+# OBFUSCATE_FIELDS_SEP = ["date", "time"]
+# # Fields to obfuscate based on length
+# OBFUSCATE_FIELDS = ["ts"]
+# OBFUSCATE_KEEP_RATIO = 0.9
+# # DON'T obfuscate the last X fields of values separates by dashes
+# OBFUSCATE_DASH_NUM = 2
+# # DON'T obfuscate the last X fields of values separates by colons
+# OBFUSCATE_COLON_NUM = 1

-SEARCH_FIELDS_DENY = ["ts", "date", "time"]
-
-DELAY_RESULTS = True
-# Delay results by this many days
-DELAY_DURATION = 10
-
-# Common to encryption and hashing
-WHITELIST_FIELDS = [
-    "ts",
-    "date",
-    "time",
-    "sentiment",
-    "version_sentiment",
-    "tokens",
-    "num_chans",
-    "num_users",
-    "online",
-    "src",
-    "exemption",
-    "hidden",
-    "type",
-]
-
-# Don't obfuscate these parameters, or lookup hashes in them
-NO_OBFUSCATE_PARAMS = [
-    "query",
-    #    "query_full",
-    "size",
-    "source",
-    "sorting",
-    "tags",
-    "index",
-    "dedup",
-    "check_sentiment",
-    "sentiment_method",
-    "dates",
-    "sort",
-    "page",
-]
-
-# Don't allow tag search for these parameters
-TAG_SEARCH_DENY = [
-    "query",
-    "query_full",
-    "size",
-    "source",
-    "sorting",
-    "tags",
-    "index",
-    "dedup",
-    "check_sentiment",
-    "sentiment_method",
-    "dates",
-    "sort",
-    "page",
-]
+# SEARCH_FIELDS_DENY = ["ts", "date", "time"]

+# DELAY_RESULTS = True
+# # Delay results by this many days
+# DELAY_DURATION = 10

 OPENSEARCH_BLACKLISTED = {}

-# URLs
+
+# URLs\
 DOMAIN = "example.com"
 URL = f"https://{DOMAIN}"

@ -121,8 +87,8 @@ SECRET_KEY = "a"
 STRIPE_ADMIN_COUPON = ""

 # Threshold
-THRESHOLD_ENDPOINT = ""
-THRESHOLD_API_KEY = "api_1"
+THRESHOLD_ENDPOINT = "http://threshold-app-1:13869"
+THRESHOLD_API_KEY = ""
 THRESHOLD_API_TOKEN = ""
 THRESHOLD_API_COUNTER = ""

@ -138,3 +104,12 @@ META_QUERY_SIZE = 10000

 DEBUG = True
 PROFILER = False
+
+if DEBUG:
+    import socket  # only if you haven't already imported this
+
+    hostname, _, ips = socket.gethostbyname_ex(socket.gethostname())
+    INTERNAL_IPS = [ip[: ip.rfind(".")] + ".1" for ip in ips] + [
+        "127.0.0.1",
+        "10.0.2.2",
+    ]
--- a/app/settings.py
+++ b/app/settings.py
@ -42,7 +42,6 @@ INSTALLED_APPS = [
    "crispy_bulma",
    "django_tables2",
    "django_tables2_bulma_template",
-
 ]
 CRISPY_TEMPLATE_PACK = "bulma"
 CRISPY_ALLOWED_TEMPLATE_PACKS = ("bulma",)
@ -149,21 +148,21 @@ INTERNAL_IPS = [
 ]

 DEBUG_TOOLBAR_PANELS = [
-    'template_profiler_panel.panels.template.TemplateProfilerPanel',
-    'debug_toolbar.panels.history.HistoryPanel',
-    'debug_toolbar.panels.versions.VersionsPanel',
-    'debug_toolbar.panels.timer.TimerPanel',
-    'debug_toolbar.panels.settings.SettingsPanel',
-    'debug_toolbar.panels.headers.HeadersPanel',
-    'debug_toolbar.panels.request.RequestPanel',
-    'debug_toolbar.panels.sql.SQLPanel',
-    'debug_toolbar.panels.staticfiles.StaticFilesPanel',
-    'debug_toolbar.panels.templates.TemplatesPanel',
-    'debug_toolbar.panels.cache.CachePanel',
-    'debug_toolbar.panels.signals.SignalsPanel',
-    'debug_toolbar.panels.logging.LoggingPanel',
-    'debug_toolbar.panels.redirects.RedirectsPanel',
-    'debug_toolbar.panels.profiling.ProfilingPanel',
+    "template_profiler_panel.panels.template.TemplateProfilerPanel",
+    "debug_toolbar.panels.history.HistoryPanel",
+    "debug_toolbar.panels.versions.VersionsPanel",
+    "debug_toolbar.panels.timer.TimerPanel",
+    "debug_toolbar.panels.settings.SettingsPanel",
+    "debug_toolbar.panels.headers.HeadersPanel",
+    "debug_toolbar.panels.request.RequestPanel",
+    "debug_toolbar.panels.sql.SQLPanel",
+    "debug_toolbar.panels.staticfiles.StaticFilesPanel",
+    "debug_toolbar.panels.templates.TemplatesPanel",
+    "debug_toolbar.panels.cache.CachePanel",
+    "debug_toolbar.panels.signals.SignalsPanel",
+    "debug_toolbar.panels.logging.LoggingPanel",
+    "debug_toolbar.panels.redirects.RedirectsPanel",
+    "debug_toolbar.panels.profiling.ProfilingPanel",
 ]

 from app.local_settings import *  # noqa
--- a/app/urls.py
+++ b/app/urls.py
@ -73,7 +73,7 @@ from core.views.ui.insights import (
 )

 urlpatterns = [
-    path('__debug__/', include('debug_toolbar.urls')),
+    path("__debug__/", include("debug_toolbar.urls")),
    path("", DrilldownTableView.as_view(), name="home"),
    path("search/", DrilldownTableView.as_view(), name="search"),
    path("about/", About.as_view(), name="about"),
--- a/core/lib/manticore.py
+++ b/core/lib/manticore.py
@ -5,7 +5,7 @@ import manticoresearch
 from django.conf import settings

 from core.lib.processing import annotate_results, filter_blacklisted, parse_results
-from core.views.helpers import dedup_list
+from core.views import helpers


 def initialise_manticore():
@ -68,7 +68,9 @@ def query_results(
    sort = None
    query_created = False
    source = None
-
+    print("BEFORE ADD DEFAULTS", query_params)
+    helpers.add_defaults(query_params)
+    print("AFTER ADD DEFAULTS", query_params)
    # Check size
    if request.user.is_anonymous:
        sizes = settings.MANTICORE_MAIN_SIZES_ANON
@ -283,7 +285,7 @@ def query_results(
    if dedup:
        if not dedup_fields:
            dedup_fields = ["msg", "nick", "ident", "host", "net", "channel"]
-        results_parsed = dedup_list(results_parsed, dedup_fields)
+        results_parsed = helpers.dedup_list(results_parsed, dedup_fields)
    context = {
        "object_list": results_parsed,
        "card": results["hits"]["total"],
--- a/core/templates/wm/widget.html
+++ b/core/templates/wm/widget.html
@ -35,4 +35,3 @@
    {% block custom_script_end %}
    {% endblock %}
 </script>
-  
--- a/core/views/helpers.py
+++ b/core/views/helpers.py
@ -9,6 +9,7 @@
 # from sortedcontainers import SortedSet

 # from core import r
+from django.conf import settings


 class SearchDenied:
@ -23,6 +24,19 @@ class LookupDenied:
        self.value = value


+def remove_defaults(query_params):
+    for field, value in list(query_params.items()):
+        if field in settings.DRILLDOWN_DEFAULT_PARAMS:
+            if value == settings.DRILLDOWN_DEFAULT_PARAMS[field]:
+                del query_params[field]
+
+
+def add_defaults(query_params):
+    for field, value in settings.DRILLDOWN_DEFAULT_PARAMS.items():
+        if field not in query_params:
+            query_params[field] = value
+
+
 def dedup_list(data, check_keys):
    """
    Remove duplicate dictionaries from list.
--- a/core/views/ui/drilldown.py
+++ b/core/views/ui/drilldown.py
@ -21,6 +21,7 @@ from core.lib.threshold import (
    get_chans,
    get_users,
 )
+from core.views import helpers
 from core.views.ui.tables import DrilldownTable

 # from copy import deepcopy
@ -155,11 +156,6 @@ def drilldown_search(request, return_context=False, template=None):
    # else:
    #     context = {"object_list": []}

-    # Remove null values
-    if "query_full" in query_params:
-        if query_params["query_full"] == "":
-            del query_params["query_full"]
-
    if "tags" in query_params:
        if query_params["tags"] == "":
            del query_params["tags"]
@ -172,7 +168,9 @@ def drilldown_search(request, return_context=False, template=None):

    # Valid sizes
    context["sizes"] = sizes
-
+    print("BEFORE REMOVE DEFAULT", query_params)
+    helpers.remove_defaults(query_params)
+    print("AFTER REMOVE DEFAULT", query_params)
    url_params = urllib.parse.urlencode(query_params)
    context["client_uri"] = url_params
    context["params"] = query_params
@ -199,17 +197,6 @@ def drilldown_search(request, return_context=False, template=None):
    clean_url_params = urllib.parse.urlencode(clean_params)
    context["uri"] = clean_url_params

-    # Warn users trying to use query string that the simple query supersedes it
-    if all([x in query_params for x in ["query", "query_full"]]):
-        context["message"] = (
-            "You are searching with both query types. "
-            "The simple query will be used. "
-            "The full query will be ignored. "
-            "Remove the text from the simple query if you wish "
-            "to use the full query."
-        )
-        context["class"] = "warning"
-
    # unique = str(uuid.uuid4())[:8]
    if return_context:
        return context
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@ -15,4 +15,4 @@ WORKDIR /code
 COPY requirements.dev.txt /code/
 RUN python -m venv /venv
 RUN . /venv/bin/activate && pip install -r requirements.dev.txt
-CMD . /venv/bin/activate && exec python manage.py runserver --nothreading 0.0.0.0:8000
+CMD . /venv/bin/activate && exec python manage.py runserver 0.0.0.0:8000