Show which fields matched

.pre-commit-config.yaml

@@ -24,8 +24,7 @@ repos:
         exclude : ^core/static/css # slow
       - id: djjs
         exclude: ^core/static/js # slow
-#  - repo: https://github.com/thibaudcolas/curlylint
-#    rev: v0.13.1
-#    hooks:
-#      - id: curlylint
-#        files: \.(html|sls)$
+  - repo: https://github.com/sirwart/ripsecrets.git
+    rev: v0.1.5
+    hooks:
+      - id: ripsecrets

Dockerfile

@@ -0,0 +1,28 @@
+# syntax=docker/dockerfile:1
+FROM python:3
+ARG OPERATION
+
+RUN useradd -d /code pathogen
+RUN mkdir -p /code
+RUN chown -R pathogen:pathogen /code
+
+RUN mkdir -p /conf/static
+RUN chown -R pathogen:pathogen /conf
+
+RUN mkdir /venv
+RUN chown pathogen:pathogen /venv
+
+USER pathogen
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
+WORKDIR /code
+COPY requirements.txt /code/
+RUN python -m venv /venv
+RUN . /venv/bin/activate && pip install -r requirements.txt
+
+# CMD . /venv/bin/activate && uwsgi --ini /conf/uwsgi.ini
+
+CMD if [ "$OPERATION" = "uwsgi" ] ; then . /venv/bin/activate && uwsgi --ini /conf/uwsgi.ini ; else . /venv/bin/activate && exec python manage.py runserver 0.0.0.0:8000; fi
+
+# CMD . /venv/bin/activate && uvicorn --reload --reload-include *.html --workers 2 --uds /var/run/socks/app.sock app.asgi:application
+# CMD . /venv/bin/activate && gunicorn -b 0.0.0.0:8000 --reload app.asgi:application -k uvicorn.workers.UvicornWorker

Makefile

@@ -0,0 +1,20 @@
+run:
+	docker-compose --env-file=stack.env up -d
+
+build:
+	docker-compose --env-file=stack.env build
+
+stop:
+	docker-compose --env-file=stack.env down
+
+log:
+	docker-compose --env-file=stack.env logs -f
+
+migrate:
+	docker-compose --env-file=stack.env run --rm app sh -c ". /venv/bin/activate && python manage.py migrate"
+
+makemigrations:
+	docker-compose --env-file=stack.env run --rm app sh -c ". /venv/bin/activate && python manage.py makemigrations"
+
+auth:
+	docker-compose --env-file=stack.env run --rm app sh -c ". /venv/bin/activate && python manage.py createsuperuser"

app/local_settings.example.py

@@ -1,40 +1,37 @@
-# OpenSearch settings
-OPENSEARCH_URL = "127.0.0.1"
-OPENSEARCH_PORT = 9200
-OPENSEARCH_TLS = True
-OPENSEARCH_USERNAME = "admin"
-OPENSEARCH_PASSWORD = ""
-
-OPENSEARCH_INDEX_MAIN = "pathogen-main"
-OPENSEARCH_INDEX_META = "pathogen-meta"
-OPENSEARCH_INDEX_INT = "pathogen-int"
-
-OPENSEARCH_MAIN_SIZES = ["20", "50", "100", "200", "400", "800"]
-OPENSEARCH_MAIN_SIZES_ANON = ["20", "50", "100"]
-OPENSEARCH_MAIN_SOURCES = ["dis", "4ch", "all"]
-OPENSEARCH_SOURCES_RESTRICTED = ["irc"]
+# Elasticsearch settings
+ELASTICSEARCH_URL = "10.1.0.1"
+ELASTICSEARCH_PORT = 9200
+ELASTICSEARCH_TLS = True
+ELASTICSEARCH_USERNAME = "admin"
+ELASTICSEARCH_PASSWORD = "secret"
 
 # Manticore settings
-MANTICORE_URL = "http://monolith-db-1:9308"
-MANTICORE_INDEX_MAIN = "main"
-MANTICORE_INDEX_META = "meta"
-MANTICORE_INDEX_INT = "internal"
+MANTICORE_URL = "http://example-db-1:9308"
 
-MANTICORE_MAIN_SIZES = ["20", "50", "100", "200", "400", "800"]
-MANTICORE_MAIN_SIZES_ANON = ["20", "50", "100"]
-MANTICORE_MAIN_SOURCES = ["dis", "4ch", "all"]
-MANTICORE_SOURCES_RESTRICTED = ["irc"]
-MANTICORE_CACHE = True
-MANTICORE_CACHE_TIMEOUT = 60
+DB_BACKEND = "ELASTICSEARCH"
+
+# Common DB settings
+INDEX_MAIN = "main"
+INDEX_RESTRICTED = "restricted"
+INDEX_META = "meta"
+INDEX_INT = "internal"
+
+MAIN_SIZES = ["1", "5", "15", "30", "50", "100", "250", "500", "1000"]
+MAIN_SIZES_ANON = ["1", "5", "15", "30", "50", "100"]
+MAIN_SOURCES = ["dis", "4ch", "all"]
+SOURCES_RESTRICTED = ["irc"]
+CACHE = False
+CACHE_TIMEOUT = 2
 
 DRILLDOWN_RESULTS_PER_PAGE = 15
 DRILLDOWN_DEFAULT_PARAMS = {
-    "size": "20",
+    "size": "15",
     "index": "main",
     "sorting": "desc",
     "source": "4ch",
 }
 
+
 # Encryption
 # ENCRYPTION = False
 # ENCRYPTION_KEY = b""
@@ -61,7 +58,7 @@ DRILLDOWN_DEFAULT_PARAMS = {
 # # Delay results by this many days
 # DELAY_DURATION = 10
 
-OPENSEARCH_BLACKLISTED = {}
+ELASTICSEARCH_BLACKLISTED = {}
 
 
 # URLs\
@@ -89,8 +86,8 @@ SECRET_KEY = "a"
 STRIPE_ADMIN_COUPON = ""
 
 # Threshold
-THRESHOLD_ENDPOINT = "http://threshold-app-1:13869"
-THRESHOLD_API_KEY = ""
+THRESHOLD_ENDPOINT = "http://threshold:13869"
+THRESHOLD_API_KEY = "api_1"
 THRESHOLD_API_TOKEN = ""
 THRESHOLD_API_COUNTER = ""
 
@@ -106,12 +103,3 @@ META_QUERY_SIZE = 10000
 
 DEBUG = True
 PROFILER = False
-
-if DEBUG:
-    import socket  # only if you haven't already imported this
-
-    hostname, _, ips = socket.gethostbyname_ex(socket.gethostname())
-    INTERNAL_IPS = [ip[: ip.rfind(".")] + ".1" for ip in ips] + [
-        "127.0.0.1",
-        "10.0.2.2",
-    ]

app/urls.py

@@ -19,8 +19,9 @@ from django.contrib import admin
 from django.urls import include, path
 from django.views.generic import TemplateView
 
+# Notification settings and rules
 # Threshold API stuff
-from core.views import About, Billing, Cancel, Order, Portal, Signup
+from core.views import About, Billing, Cancel, Order, Portal, Signup, notifications
 from core.views.callbacks import Callback
 from core.views.manage.threshold.irc import (
     ThresholdIRCNetworkList,  # Actions and just get list output
@@ -77,6 +78,7 @@ urlpatterns = [
     path("__debug__/", include("debug_toolbar.urls")),
     path("", DrilldownTableView.as_view(), name="home"),
     path("search/", DrilldownTableView.as_view(), name="search"),
+    path("search/partial/", DrilldownTableView.as_view(), name="search_partial"),
     path("about/", About.as_view(), name="about"),
     path("callback", Callback.as_view(), name="callback"),
     path("billing/", Billing.as_view(), name="billing"),
@@ -260,4 +262,29 @@ urlpatterns = [
         name="threshold_irc_msg",
     ),
     ##
+    path(
+        "notifications/<str:type>/update/",
+        notifications.NotificationsUpdate.as_view(),
+        name="notifications_update",
+    ),
+    path(
+        "rules/<str:type>/",
+        notifications.RuleList.as_view(),
+        name="rules",
+    ),
+    path(
+        "rule/<str:type>/create/",
+        notifications.RuleCreate.as_view(),
+        name="rule_create",
+    ),
+    path(
+        "rule/<str:type>/update/<str:pk>/",
+        notifications.RuleUpdate.as_view(),
+        name="rule_update",
+    ),
+    path(
+        "rule/<str:type>/delete/<str:pk>/",
+        notifications.RuleDelete.as_view(),
+        name="rule_delete",
+    ),
 ] + static(settings.STATIC_URL, document_root=settings.STATIC_ROOT)

core/__init__.py

@@ -2,7 +2,7 @@ import stripe
 from django.conf import settings
 from redis import StrictRedis
 
-r = StrictRedis(unix_socket_path="/var/run/redis/redis.sock", db=0)
+r = StrictRedis(unix_socket_path="/var/run/socks/redis.sock", db=0)
 
 if settings.STRIPE_TEST:
     stripe.api_key = settings.STRIPE_API_KEY_TEST

core/db/__init__.py

@@ -1,6 +1,7 @@
 import random
 import string
 import time
+from abc import ABC, abstractmethod
 from datetime import datetime
 from math import floor, log10
 
@@ -11,10 +12,49 @@ from siphashc import siphash
 from core import r
 from core.db.processing import annotate_results
 from core.util import logs
-from core.views import helpers
 
 
-class StorageBackend(object):
+def remove_defaults(query_params):
+    for field, value in list(query_params.items()):
+        if field in settings.DRILLDOWN_DEFAULT_PARAMS:
+            if value == settings.DRILLDOWN_DEFAULT_PARAMS[field]:
+                del query_params[field]
+
+
+def add_defaults(query_params):
+    for field, value in settings.DRILLDOWN_DEFAULT_PARAMS.items():
+        if field not in query_params:
+            query_params[field] = value
+
+
+def dedup_list(data, check_keys):
+    """
+    Remove duplicate dictionaries from list.
+    """
+    seen = set()
+    out = []
+
+    dup_count = 0
+    for x in data:
+        dedupeKey = tuple(x[k] for k in check_keys if k in x)
+        if dedupeKey in seen:
+            dup_count += 1
+            continue
+        if dup_count > 0:
+            out.append({"type": "control", "hidden": dup_count})
+            dup_count = 0
+        out.append(x)
+        seen.add(dedupeKey)
+    if dup_count > 0:
+        out.append({"type": "control", "hidden": dup_count})
+    return out
+
+
+class QueryError(Exception):
+    pass
+
+
+class StorageBackend(ABC):
     def __init__(self, name):
         self.log = logs.get_logger(name)
         self.log.info(f"Initialising storage backend {name}")
@@ -22,8 +62,9 @@ class StorageBackend(object):
         self.initialise_caching()
         self.initialise()
 
+    @abstractmethod
     def initialise(self, **kwargs):
-        raise NotImplementedError
+        pass
 
     def initialise_caching(self):
         hash_key = r.get("cache_hash_key")
@@ -37,11 +78,13 @@ class StorageBackend(object):
             self.log.debug(f"Decoded hash key: {hash_key}")
         self.hash_key = hash_key
 
+    @abstractmethod
     def construct_query(self, **kwargs):
-        raise NotImplementedError
+        pass
 
+    @abstractmethod
     def run_query(self, **kwargs):
-        raise NotImplementedError
+        pass
 
     def parse_size(self, query_params, sizes):
         if "size" in query_params:
@@ -56,14 +99,16 @@ class StorageBackend(object):
 
         return size
 
-    def parse_index(self, user, query_params):
+    def parse_index(self, user, query_params, raise_error=False):
         if "index" in query_params:
             index = query_params["index"]
             if index == "main":
                 index = settings.INDEX_MAIN
             else:
                 if not user.has_perm(f"core.index_{index}"):
-                    message = "Not permitted to search by this index"
+                    message = f"Not permitted to search by this index: {index}"
+                    if raise_error:
+                        raise QueryError(message)
                     message_class = "danger"
                     return {
                         "message": message,
@@ -75,7 +120,9 @@ class StorageBackend(object):
                     index = settings.INDEX_INT
                 elif index == "restricted":
                     if not user.has_perm("core.restricted_sources"):
-                        message = "Not permitted to search by this index"
+                        message = f"Not permitted to search by this index: {index}"
+                        if raise_error:
+                            raise QueryError(message)
                         message_class = "danger"
                         return {
                             "message": message,
@@ -83,7 +130,9 @@ class StorageBackend(object):
                         }
                     index = settings.INDEX_RESTRICTED
                 else:
-                    message = "Index is not valid."
+                    message = f"Index is not valid: {index}"
+                    if raise_error:
+                        raise QueryError(message)
                     message_class = "danger"
                     return {
                         "message": message,
@@ -93,22 +142,22 @@ class StorageBackend(object):
             index = settings.INDEX_MAIN
         return index
 
-    def parse_query(self, query_params, tags, size, index, custom_query, add_bool):
+    def parse_query(self, query_params, tags, size, custom_query, add_bool, **kwargs):
         query_created = False
         if "query" in query_params:
             query = query_params["query"]
-            search_query = self.construct_query(query, size, index)
+            search_query = self.construct_query(query, size, **kwargs)
             query_created = True
         else:
             if custom_query:
                 search_query = custom_query
             else:
-                search_query = self.construct_query(None, size, index, blank=True)
+                search_query = self.construct_query(None, size, blank=True, **kwargs)
 
         if tags:
             # Get a blank search query
             if not query_created:
-                search_query = self.construct_query(None, size, index, blank=True)
+                search_query = self.construct_query(None, size, blank=True, **kwargs)
                 query_created = True
             for item in tags:
                 for tagname, tagvalue in item.items():
@@ -128,17 +177,22 @@ class StorageBackend(object):
                 message_class = "warning"
                 return {"message": message, "class": message_class}
 
-    def parse_source(self, user, query_params):
+    def parse_source(self, user, query_params, raise_error=False):
+        source = None
         if "source" in query_params:
             source = query_params["source"]
 
             if source in settings.SOURCES_RESTRICTED:
                 if not user.has_perm("core.restricted_sources"):
-                    message = "Access denied"
+                    message = f"Access denied: {source}"
+                    if raise_error:
+                        raise QueryError(message)
                     message_class = "danger"
                     return {"message": message, "class": message_class}
             elif source not in settings.MAIN_SOURCES:
-                message = "Invalid source"
+                message = f"Invalid source: {source}"
+                if raise_error:
+                    raise QueryError(message)
                 message_class = "danger"
                 return {"message": message, "class": message_class}
 
@@ -217,7 +271,7 @@ class StorageBackend(object):
         # For every hit from ES
         for index, item in enumerate(list(response["hits"]["hits"])):
             # For every blacklisted type
-            for blacklisted_type in settings.OPENSEARCH_BLACKLISTED.keys():
+            for blacklisted_type in settings.ELASTICSEARCH_BLACKLISTED.keys():
                 # Check this field we are matching exists
                 if "_source" in item.keys():
                     data_index = "_source"
@@ -228,9 +282,7 @@ class StorageBackend(object):
                 if blacklisted_type in item[data_index].keys():
                     content = item[data_index][blacklisted_type]
                     # For every item in the blacklisted array for the type
-                    for blacklisted_item in settings.OPENSEARCH_BLACKLISTED[
-                        blacklisted_type
-                    ]:
+                    for blacklisted_item in settings.BLACKLISTED[blacklisted_type]:
                         if blacklisted_item == str(content):
                             # Remove the item
                             if item in response["hits"]["hits"]:
@@ -255,7 +307,7 @@ class StorageBackend(object):
         # Actually get rid of all the things we set to None
         response["hits"]["hits"] = [hit for hit in response["hits"]["hits"] if hit]
 
-    def query(self, user, search_query):
+    def query(self, user, search_query, **kwargs):
         # For time tracking
         start = time.process_time()
         if settings.CACHE:
@@ -265,8 +317,6 @@ class StorageBackend(object):
             cache_hit = r.get(f"query_cache.{user.id}.{hash}")
             if cache_hit:
                 response = orjson.loads(cache_hit)
-                print("CACHE HIT", response)
-
                 time_took = (time.process_time() - start) * 1000
                 # Round to 3 significant figures
                 time_took_rounded = round(
@@ -277,7 +327,19 @@ class StorageBackend(object):
                     "took": time_took_rounded,
                     "cache": True,
                 }
-        response = self.run_query(user, search_query)
+        response = self.run_query(user, search_query, **kwargs)
+
+        # For Elasticsearch
+        if isinstance(response, Exception):
+            message = f"Error: {response.info['error']['root_cause'][0]['type']}"
+            message_class = "danger"
+            return {"message": message, "class": message_class}
+        if len(response["hits"]["hits"]) == 0:
+            message = "No results."
+            message_class = "danger"
+            return {"message": message, "class": message_class}
+
+        # For Druid
         if "error" in response:
             if "errorMessage" in response:
                 context = {
@@ -287,12 +349,12 @@ class StorageBackend(object):
                 return context
             else:
                 return response
-        # response = response.to_dict()
-        # print("RESP", response)
         if "took" in response:
             if response["took"] is None:
                 return None
-        self.filter_blacklisted(user, response)
+
+        # Removed for now, no point given we have restricted indexes
+        # self.filter_blacklisted(user, response)
 
         # Parse the response
         response_parsed = self.parse(response)
@@ -308,18 +370,22 @@ class StorageBackend(object):
         time_took_rounded = round(time_took, 3 - int(floor(log10(abs(time_took)))) - 1)
         return {"object_list": response_parsed, "took": time_took_rounded}
 
+    @abstractmethod
     def query_results(self, **kwargs):
-        raise NotImplementedError
+        pass
 
     def process_results(self, response, **kwargs):
         if kwargs.get("annotate"):
             annotate_results(response)
+        if kwargs.get("reverse"):
+            response.reverse()
         if kwargs.get("dedup"):
-            response = response[::-1]
-        if kwargs.get("dedup"):
-            if not kwargs.get("dedup_fields"):
+            dedup_fields = kwargs.get("dedup_fields")
+            if not dedup_fields:
                 dedup_fields = ["msg", "nick", "ident", "host", "net", "channel"]
-            response = helpers.dedup_list(response, dedup_fields)
+            response = dedup_list(response, dedup_fields)
+        return response
 
+    @abstractmethod
     def parse(self, response):
-        raise NotImplementedError
+        pass

core/db/druid.py

@@ -4,9 +4,8 @@ import orjson
 import requests
 from django.conf import settings
 
-from core.db import StorageBackend
+from core.db import StorageBackend, add_defaults
 from core.db.processing import parse_druid
-from core.views import helpers
 
 logger = logging.getLogger(__name__)
 
@@ -77,7 +76,8 @@ class DruidBackend(StorageBackend):
             self.add_type("or", search_query, extra_should2)
         return search_query
 
-    def construct_query(self, query, size, index, blank=False):
+    def construct_query(self, query, size, blank=False, **kwargs):
+        index = kwargs.get("index")
         search_query = {
             "limit": size,
             "queryType": "scan",
@@ -107,19 +107,13 @@ class DruidBackend(StorageBackend):
 
     def parse(self, response):
         parsed = parse_druid(response)
-        print("PARSE LEN", len(parsed))
         return parsed
 
     def run_query(self, user, search_query):
         ss = orjson.dumps(search_query, option=orjson.OPT_INDENT_2)
         ss = ss.decode()
-        print(ss)
-        response = requests.post("http://broker:8082/druid/v2", json=search_query)
+        response = requests.post("http://druid:8082/druid/v2", json=search_query)
         response = orjson.loads(response.text)
-        print("RESPONSE LEN", len(response))
-        # ss = orjson.dumps(response, option=orjson.OPT_INDENT_2)
-        # ss = ss.decode()
-        # print(ss)
         return response
 
     def filter_blacklisted(self, user, response):
@@ -140,7 +134,7 @@ class DruidBackend(StorageBackend):
         add_bool = []
         add_in = {}
 
-        helpers.add_defaults(query_params)
+        add_defaults(query_params)
 
         # Now, run the helpers for SIQTSRSS/ADR
         # S - Size
@@ -172,7 +166,7 @@ class DruidBackend(StorageBackend):
 
         # Q/T - Query/Tags
         search_query = self.parse_query(
-            query_params, tags, size, index, custom_query, add_bool
+            query_params, tags, size, custom_query, add_bool, index=index
         )
         # Query should be a dict, so check if it contains message here
         if "message" in search_query:
@@ -232,18 +226,13 @@ class DruidBackend(StorageBackend):
         response = self.query(request.user, search_query)
 
         # A/D/R - Annotate/Dedup/Reverse
-        self.process_results(
+        response = self.process_results(
             response,
             annotate=annotate,
             dedup=dedup,
             dedup_fields=dedup_fields,
             reverse=reverse,
         )
-        # ss = orjson.dumps(list(response), option=orjson.OPT_INDENT_2)
-        # ss = ss.decode()
-        # print(ss)
-        # print("PARSED", results_parsed)
-        # return results_parsed
         context = response
         return context
 

core/db/elastic.py

@@ -0,0 +1,374 @@
+# from copy import deepcopy
+# from datetime import datetime, timedelta
+
+from django.conf import settings
+from elasticsearch import Elasticsearch
+from elasticsearch.exceptions import NotFoundError, RequestError
+
+from core.db import StorageBackend, add_defaults
+
+# from json import dumps
+# pp = lambda x: print(dumps(x, indent=2))
+from core.db.processing import parse_results
+
+
+class ElasticsearchBackend(StorageBackend):
+    def __init__(self):
+        super().__init__("Elasticsearch")
+
+    def initialise(self, **kwargs):
+        """
+        Inititialise the Elastuicsearch API endpoint.
+        """
+        auth = (settings.ELASTICSEARCH_USERNAME, settings.ELASTICSEARCH_PASSWORD)
+        client = Elasticsearch(
+            settings.ELASTICSEARCH_URL, http_auth=auth, verify_certs=False
+        )
+        self.client = client
+
+    def construct_context_query(
+        self, index, net, channel, src, num, size, type=None, nicks=None
+    ):
+        # Get the initial query
+        query = self.construct_query(None, size, blank=True)
+
+        extra_must = []
+        extra_should = []
+        extra_should2 = []
+        if num:
+            extra_must.append({"match_phrase": {"num": num}})
+        if net:
+            extra_must.append({"match_phrase": {"net": net}})
+        if channel:
+            extra_must.append({"match": {"channel": channel}})
+        if nicks:
+            for nick in nicks:
+                extra_should2.append({"match": {"nick": nick}})
+
+        types = ["msg", "notice", "action", "kick", "topic", "mode"]
+        fields = [
+            "nick",
+            "ident",
+            "host",
+            "channel",
+            "ts",
+            "msg",
+            "type",
+            "net",
+            "src",
+            "tokens",
+        ]
+        query["fields"] = fields
+
+        if index == "internal":
+            fields.append("mtype")
+            if channel == "*status" or type == "znc":
+                if {"match": {"channel": channel}} in extra_must:
+                    extra_must.remove({"match": {"channel": channel}})
+                extra_should2 = []
+                # Type is one of msg or notice
+                # extra_should.append({"match": {"mtype": "msg"}})
+                # extra_should.append({"match": {"mtype": "notice"}})
+                extra_should.append({"match": {"type": "znc"}})
+                extra_should.append({"match": {"type": "self"}})
+
+                extra_should2.append({"match": {"type": "znc"}})
+                extra_should2.append({"match": {"nick": channel}})
+            elif type == "auth":
+                if {"match": {"channel": channel}} in extra_must:
+                    extra_must.remove({"match": {"channel": channel}})
+                extra_should2 = []
+                extra_should2.append({"match": {"nick": channel}})
+                # extra_should2.append({"match": {"mtype": "msg"}})
+                # extra_should2.append({"match": {"mtype": "notice"}})
+
+                extra_should.append({"match": {"type": "query"}})
+                extra_should2.append({"match": {"type": "self"}})
+                extra_should.append({"match": {"nick": channel}})
+            else:
+                for ctype in types:
+                    extra_should.append({"equals": {"mtype": ctype}})
+        else:
+            for ctype in types:
+                extra_should.append({"match": {"type": ctype}})
+        # query = {
+        #     "index": index,
+        #     "limit": size,
+        #     "query": {
+        #         "bool": {
+        #             "must": [
+        #                 # {"equals": {"src": src}},
+        #                 # {
+        #                 #     "bool": {
+        #                 #         "should": [*extra_should],
+        #                 #     }
+        #                 # },
+        #                 # {
+        #                 #     "bool": {
+        #                 #         "should": [*extra_should2],
+        #                 #     }
+        #                 # },
+        #                 *extra_must,
+        #             ]
+        #         }
+        #     },
+        #     "fields": fields,
+        #     # "_source": False,
+        # }
+        if extra_must:
+            for x in extra_must:
+                query["query"]["bool"]["must"].append(x)
+        if extra_should:
+            query["query"]["bool"]["must"].append({"bool": {"should": [*extra_should]}})
+        if extra_should2:
+            query["query"]["bool"]["must"].append(
+                {"bool": {"should": [*extra_should2]}}
+            )
+        return query
+
+    def construct_query(self, query, size, blank=False):
+        """
+        Accept some query parameters and construct an Elasticsearch query.
+        """
+        query_base = {
+            "size": size,
+            "query": {"bool": {"must": []}},
+        }
+        query_string = {
+            "query_string": {
+                "query": query,
+                # "fields": fields,
+                # "default_field": "msg",
+                # "type": "best_fields",
+                "fuzziness": "AUTO",
+                "fuzzy_transpositions": True,
+                "fuzzy_max_expansions": 50,
+                "fuzzy_prefix_length": 0,
+                # "minimum_should_match": 1,
+                "default_operator": "and",
+                "analyzer": "standard",
+                "lenient": True,
+                "boost": 1,
+                "allow_leading_wildcard": True,
+                # "enable_position_increments": False,
+                "phrase_slop": 3,
+                # "max_determinized_states": 10000,
+                "quote_field_suffix": "",
+                "quote_analyzer": "standard",
+                "analyze_wildcard": False,
+                "auto_generate_synonyms_phrase_query": True,
+            }
+        }
+        if not blank:
+            query_base["query"]["bool"]["must"].append(query_string)
+        return query_base
+
+    def parse(self, response):
+        parsed = parse_results(response)
+        return parsed
+
+    def run_query(self, user, search_query, **kwargs):
+        """
+        Low level helper to run an ES query.
+        Accept a user to pass it to the filter, so we can
+        avoid filtering for superusers.
+        Accept fields and size, for the fields we want to match and the
+        number of results to return.
+        """
+        index = kwargs.get("index")
+        try:
+            response = self.client.search(body=search_query, index=index)
+        except RequestError as err:
+            print("Elasticsearch error", err)
+            return err
+        except NotFoundError as err:
+            print("Elasticsearch error", err)
+            return err
+        return response
+
+    def query_results(
+        self,
+        request,
+        query_params,
+        size=None,
+        annotate=True,
+        custom_query=False,
+        reverse=False,
+        dedup=False,
+        dedup_fields=None,
+        tags=None,
+    ):
+
+        add_bool = []
+        add_top = []
+        add_top_negative = []
+
+        add_defaults(query_params)
+
+        # Now, run the helpers for SIQTSRSS/ADR
+        # S - Size
+        # I - Index
+        # Q - Query
+        # T - Tags
+        # S - Source
+        # R - Ranges
+        # S - Sort
+        # S - Sentiment
+        # A - Annotate
+        # D - Dedup
+        # R - Reverse
+
+        # S - Size
+        if request.user.is_anonymous:
+            sizes = settings.MAIN_SIZES_ANON
+        else:
+            sizes = settings.MAIN_SIZES
+        if not size:
+            size = self.parse_size(query_params, sizes)
+            if isinstance(size, dict):
+                return size
+
+        # I - Index
+        index = self.parse_index(request.user, query_params)
+        if isinstance(index, dict):
+            return index
+
+        # Q/T - Query/Tags
+        search_query = self.parse_query(
+            query_params, tags, size, custom_query, add_bool
+        )
+        # Query should be a dict, so check if it contains message here
+        if "message" in search_query:
+            return search_query
+
+        # S - Sources
+        sources = self.parse_source(request.user, query_params)
+        if isinstance(sources, dict):
+            return sources
+        total_count = len(sources)
+        total_sources = len(settings.MAIN_SOURCES) + len(settings.SOURCES_RESTRICTED)
+        if total_count != total_sources:
+            add_top_tmp = {"bool": {"should": []}}
+            for source_iter in sources:
+                add_top_tmp["bool"]["should"].append(
+                    {"match_phrase": {"src": source_iter}}
+                )
+            add_top.append(add_top_tmp)
+
+        # R - Ranges
+        # date_query = False
+        from_ts, to_ts = self.parse_date_time(query_params)
+        if from_ts:
+            range_query = {
+                "range": {
+                    "ts": {
+                        "gt": from_ts,
+                        "lt": to_ts,
+                    }
+                }
+            }
+            add_top.append(range_query)
+
+        # S - Sort
+        sort = self.parse_sort(query_params)
+        if isinstance(sort, dict):
+            return sort
+        if sort:
+            # For Druid compatibility
+            sort_map = {"ascending": "asc", "descending": "desc"}
+            sorting = [
+                {
+                    "ts": {
+                        "order": sort_map[sort],
+                    }
+                }
+            ]
+            search_query["sort"] = sorting
+
+        # S - Sentiment
+        sentiment_r = self.parse_sentiment(query_params)
+        if isinstance(sentiment_r, dict):
+            return sentiment_r
+        if sentiment_r:
+            sentiment_method, sentiment = sentiment_r
+            range_query_compare = {"range": {"sentiment": {}}}
+            range_query_precise = {
+                "match": {
+                    "sentiment": None,
+                }
+            }
+            if sentiment_method == "below":
+                range_query_compare["range"]["sentiment"]["lt"] = sentiment
+                add_top.append(range_query_compare)
+            elif sentiment_method == "above":
+                range_query_compare["range"]["sentiment"]["gt"] = sentiment
+                add_top.append(range_query_compare)
+            elif sentiment_method == "exact":
+                range_query_precise["match"]["sentiment"] = sentiment
+                add_top.append(range_query_precise)
+            elif sentiment_method == "nonzero":
+                range_query_precise["match"]["sentiment"] = 0
+                add_top_negative.append(range_query_precise)
+
+        # Add in the additional information we already populated
+        self.add_bool(search_query, add_bool)
+        self.add_top(search_query, add_top)
+        self.add_top(search_query, add_top_negative, negative=True)
+
+        response = self.query(
+            request.user,
+            search_query,
+            index=index,
+        )
+        if "message" in response:
+            return response
+
+        # A/D/R - Annotate/Dedup/Reverse
+        response["object_list"] = self.process_results(
+            response["object_list"],
+            annotate=annotate,
+            dedup=dedup,
+            dedup_fields=dedup_fields,
+            reverse=reverse,
+        )
+
+        context = response
+        return context
+
+    def query_single_result(self, request, query_params):
+        context = self.query_results(request, query_params, size=100)
+
+        if not context:
+            return {"message": "Failed to run query", "message_class": "danger"}
+        if "message" in context:
+            return context
+        dedup_set = {item["nick"] for item in context["object_list"]}
+        if dedup_set:
+            context["item"] = context["object_list"][0]
+
+        return context
+
+    def add_bool(self, search_query, add_bool):
+        """
+        Add the specified boolean matches to search query.
+        """
+        if not add_bool:
+            return
+        for item in add_bool:
+            search_query["query"]["bool"]["must"].append({"match_phrase": item})
+
+    def add_top(self, search_query, add_top, negative=False):
+        """
+        Merge add_top with the base of the search_query.
+        """
+        if not add_top:
+            return
+        if negative:
+            for item in add_top:
+                if "must_not" in search_query["query"]["bool"]:
+                    search_query["query"]["bool"]["must_not"].append(item)
+                else:
+                    search_query["query"]["bool"]["must_not"] = [item]
+        else:
+            for item in add_top:
+                search_query["query"]["bool"]["must"].append(item)

core/db/manticore.py

@@ -5,9 +5,8 @@ from pprint import pprint
 import requests
 from django.conf import settings
 
-from core.db import StorageBackend
+from core.db import StorageBackend, add_defaults, dedup_list
 from core.db.processing import annotate_results, parse_results
-from core.views import helpers
 
 logger = logging.getLogger(__name__)
 
@@ -67,7 +66,7 @@ class ManticoreBackend(StorageBackend):
         sort = None
         query_created = False
         source = None
-        helpers.add_defaults(query_params)
+        add_defaults(query_params)
         # Check size
         if request.user.is_anonymous:
             sizes = settings.MANTICORE_MAIN_SIZES_ANON
@@ -292,7 +291,7 @@ class ManticoreBackend(StorageBackend):
         if dedup:
             if not dedup_fields:
                 dedup_fields = ["msg", "nick", "ident", "host", "net", "channel"]
-            results_parsed = helpers.dedup_list(results_parsed, dedup_fields)
+            results_parsed = dedup_list(results_parsed, dedup_fields)
         context = {
             "object_list": results_parsed,
             "card": results["hits"]["total"],

core/db/opensearch.py

@@ -1,485 +0,0 @@
-# from copy import deepcopy
-# from datetime import datetime, timedelta
-
-from django.conf import settings
-from opensearchpy import OpenSearch
-from opensearchpy.exceptions import NotFoundError, RequestError
-
-from core.db import StorageBackend
-
-# from json import dumps
-# pp = lambda x: print(dumps(x, indent=2))
-from core.db.processing import annotate_results, parse_results
-from core.views.helpers import dedup_list
-
-
-class OpensearchBackend(StorageBackend):
-    def __init__(self):
-        super().__init__("Opensearch")
-
-    def initialise(self, **kwargs):
-        """
-        Inititialise the OpenSearch API endpoint.
-        """
-        auth = (settings.OPENSEARCH_USERNAME, settings.OPENSEARCH_PASSWORD)
-        client = OpenSearch(
-            # fmt: off
-            hosts=[{"host": settings.OPENSEARCH_URL,
-                    "port": settings.OPENSEARCH_PORT}],
-            http_compress=False,  # enables gzip compression for request bodies
-            http_auth=auth,
-            # client_cert = client_cert_path,
-            # client_key = client_key_path,
-            use_ssl=settings.OPENSEARCH_TLS,
-            verify_certs=False,
-            ssl_assert_hostname=False,
-            ssl_show_warn=False,
-            # a_certs=ca_certs_path,
-        )
-        self.client = client
-
-    def construct_query(self, query, size, use_query_string=True, tokens=False):
-        """
-        Accept some query parameters and construct an OpenSearch query.
-        """
-        if not size:
-            size = 5
-        query_base = {
-            "size": size,
-            "query": {"bool": {"must": []}},
-        }
-        query_string = {
-            "query_string": {
-                "query": query,
-                # "fields": fields,
-                # "default_field": "msg",
-                # "type": "best_fields",
-                "fuzziness": "AUTO",
-                "fuzzy_transpositions": True,
-                "fuzzy_max_expansions": 50,
-                "fuzzy_prefix_length": 0,
-                # "minimum_should_match": 1,
-                "default_operator": "or",
-                "analyzer": "standard",
-                "lenient": True,
-                "boost": 1,
-                "allow_leading_wildcard": True,
-                # "enable_position_increments": False,
-                "phrase_slop": 3,
-                # "max_determinized_states": 10000,
-                "quote_field_suffix": "",
-                "quote_analyzer": "standard",
-                "analyze_wildcard": False,
-                "auto_generate_synonyms_phrase_query": True,
-            }
-        }
-        query_tokens = {
-            "simple_query_string": {
-                # "tokens": query,
-                "query": query,
-                "fields": ["tokens"],
-                "flags": "ALL",
-                "fuzzy_transpositions": True,
-                "fuzzy_max_expansions": 50,
-                "fuzzy_prefix_length": 0,
-                "default_operator": "and",
-                "analyzer": "standard",
-                "lenient": True,
-                "boost": 1,
-                "quote_field_suffix": "",
-                "analyze_wildcard": False,
-                "auto_generate_synonyms_phrase_query": False,
-            }
-        }
-        if tokens:
-            query_base["query"]["bool"]["must"].append(query_tokens)
-            # query["query"]["bool"]["must"].append(query_string)
-            # query["query"]["bool"]["must"][0]["query_string"]["fields"] = ["tokens"]
-        elif use_query_string:
-            query_base["query"]["bool"]["must"].append(query_string)
-        return query_base
-
-    def run_query(self, client, user, query, custom_query=False, index=None, size=None):
-        """
-        Low level helper to run an ES query.
-        Accept a user to pass it to the filter, so we can
-        avoid filtering for superusers.
-        Accept fields and size, for the fields we want to match and the
-        number of results to return.
-        """
-        if not index:
-            index = settings.INDEX_MAIN
-        if custom_query:
-            search_query = query
-        else:
-            search_query = self.construct_query(query, size)
-        try:
-            response = client.search(body=search_query, index=index)
-        except RequestError as err:
-            print("OpenSearch error", err)
-            return err
-        except NotFoundError as err:
-            print("OpenSearch error", err)
-            return err
-        return response
-
-    def query_results(
-        self,
-        request,
-        query_params,
-        size=None,
-        annotate=True,
-        custom_query=False,
-        reverse=False,
-        dedup=False,
-        dedup_fields=None,
-        lookup_hashes=True,
-        tags=None,
-    ):
-        """
-        API helper to alter the OpenSearch return format into something
-        a bit better to parse.
-        Accept a HTTP request object. Run the query, and annotate the
-        results with the other data we have.
-        """
-        # is_anonymous = isinstance(request.user, AnonymousUser)
-        query = None
-        message = None
-        message_class = None
-        add_bool = []
-        add_top = []
-        add_top_negative = []
-        sort = None
-        query_created = False
-
-        # Lookup the hash values but don't disclose them to the user
-        # denied = []
-        # if lookup_hashes:
-        #     if settings.HASHING:
-        #         query_params = deepcopy(query_params)
-        #         denied_q = hash_lookup(request.user, query_params)
-        #         denied.extend(denied_q)
-        #         if tags:
-        #             denied_t = hash_lookup(request.user, tags, query_params)
-        #             denied.extend(denied_t)
-
-        # message = "Permission denied: "
-        # for x in denied:
-        #     if isinstance(x, SearchDenied):
-        #         message += f"Search({x.key}: {x.value}) "
-        #     elif isinstance(x, LookupDenied):
-        #         message += f"Lookup({x.key}: {x.value}) "
-        # if denied:
-        #     # message = [f"{i}" for i in message]
-        #     # message = "\n".join(message)
-        #     message_class = "danger"
-        #     return {"message": message, "class": message_class}
-
-        if request.user.is_anonymous:
-            sizes = settings.MAIN_SIZES_ANON
-        else:
-            sizes = settings.MAIN_SIZES
-        if not size:
-            if "size" in query_params:
-                size = query_params["size"]
-                if size not in sizes:
-                    message = "Size is not permitted"
-                    message_class = "danger"
-                    return {"message": message, "class": message_class}
-            else:
-                size = 20
-        source = None
-        if "source" in query_params:
-            source = query_params["source"]
-
-            if source in settings.SOURCES_RESTRICTED:
-                if not request.user.has_perm("core.restricted_sources"):
-                    message = "Access denied"
-                    message_class = "danger"
-                    return {"message": message, "class": message_class}
-            elif source not in settings.MAIN_SOURCES:
-                message = "Invalid source"
-                message_class = "danger"
-                return {"message": message, "class": message_class}
-
-            if source == "all":
-                source = None  # the next block will populate it
-
-        if source:
-            sources = [source]
-        else:
-            sources = settings.MAIN_SOURCES
-            if request.user.has_perm("core.restricted_sources"):
-                for source_iter in settings.SOURCES_RESTRICTED:
-                    sources.append(source_iter)
-
-        add_top_tmp = {"bool": {"should": []}}
-        for source_iter in sources:
-            add_top_tmp["bool"]["should"].append({"match_phrase": {"src": source_iter}})
-        add_top.append(add_top_tmp)
-
-        # date_query = False
-        if set({"from_date", "to_date", "from_time", "to_time"}).issubset(
-            query_params.keys()
-        ):
-            from_ts = f"{query_params['from_date']}T{query_params['from_time']}Z"
-            to_ts = f"{query_params['to_date']}T{query_params['to_time']}Z"
-            range_query = {
-                "range": {
-                    "ts": {
-                        "gt": from_ts,
-                        "lt": to_ts,
-                    }
-                }
-            }
-            add_top.append(range_query)
-
-        # if date_query:
-        #     if settings.DELAY_RESULTS:
-        #         if source not in settings.SAFE_SOURCES:
-        #             if request.user.has_perm("core.bypass_delay"):
-        #                 add_top.append(range_query)
-        #             else:
-        #                 delay_as_ts = datetime.now() - timedelta(
-        #                     days=settings.DELAY_DURATION
-        #                 )
-        #                 lt_as_ts = datetime.strptime(
-        #                     range_query["range"]["ts"]["lt"], "%Y-%m-%dT%H:%MZ"
-        #                 )
-        #                 if lt_as_ts > delay_as_ts:
-        #                     range_query["range"]["ts"][
-        #                         "lt"
-        #                     ] = f"now-{settings.DELAY_DURATION}d"
-        #                 add_top.append(range_query)
-        #     else:
-        #         add_top.append(range_query)
-        # else:
-        #     if settings.DELAY_RESULTS:
-        #         if source not in settings.SAFE_SOURCES:
-        #             if not request.user.has_perm("core.bypass_delay"):
-        #                 range_query = {
-        #                     "range": {
-        #                         "ts": {
-        #                             # "gt": ,
-        #                             "lt": f"now-{settings.DELAY_DURATION}d",
-        #                         }
-        #                     }
-        #                 }
-        #                 add_top.append(range_query)
-
-        if "sorting" in query_params:
-            sorting = query_params["sorting"]
-            if sorting not in ("asc", "desc", "none"):
-                message = "Invalid sort"
-                message_class = "danger"
-                return {"message": message, "class": message_class}
-            if sorting in ("asc", "desc"):
-                sort = [
-                    {
-                        "ts": {
-                            "order": sorting,
-                        }
-                    }
-                ]
-
-        if "check_sentiment" in query_params:
-            if "sentiment_method" not in query_params:
-                message = "No sentiment method"
-                message_class = "danger"
-                return {"message": message, "class": message_class}
-            if "sentiment" in query_params:
-                sentiment = query_params["sentiment"]
-                try:
-                    sentiment = float(sentiment)
-                except ValueError:
-                    message = "Sentiment is not a float"
-                    message_class = "danger"
-                    return {"message": message, "class": message_class}
-            sentiment_method = query_params["sentiment_method"]
-            range_query_compare = {"range": {"sentiment": {}}}
-            range_query_precise = {
-                "match": {
-                    "sentiment": None,
-                }
-            }
-            if sentiment_method == "below":
-                range_query_compare["range"]["sentiment"]["lt"] = sentiment
-                add_top.append(range_query_compare)
-            elif sentiment_method == "above":
-                range_query_compare["range"]["sentiment"]["gt"] = sentiment
-                add_top.append(range_query_compare)
-            elif sentiment_method == "exact":
-                range_query_precise["match"]["sentiment"] = sentiment
-                add_top.append(range_query_precise)
-            elif sentiment_method == "nonzero":
-                range_query_precise["match"]["sentiment"] = 0
-                add_top_negative.append(range_query_precise)
-
-        # Only one of query or query_full can be active at once
-        # We prefer query because it's simpler
-        if "query" in query_params:
-            query = query_params["query"]
-            search_query = self.construct_query(query, size, tokens=True)
-            query_created = True
-        elif "query_full" in query_params:
-            query_full = query_params["query_full"]
-            # if request.user.has_perm("core.query_search"):
-            search_query = self.construct_query(query_full, size)
-            query_created = True
-            # else:
-            #     message = "You cannot search by query string"
-            #     message_class = "danger"
-            #     return {"message": message, "class": message_class}
-        else:
-            if custom_query:
-                search_query = custom_query
-
-        if tags:
-            # Get a blank search query
-            if not query_created:
-                search_query = self.construct_query(None, size, use_query_string=False)
-                query_created = True
-            for tagname, tagvalue in tags.items():
-                add_bool.append({tagname: tagvalue})
-
-        required_any = ["query_full", "query", "tags"]
-        if not any([field in query_params.keys() for field in required_any]):
-            if not custom_query:
-                message = "Empty query!"
-                message_class = "warning"
-                return {"message": message, "class": message_class}
-
-        if add_bool:
-            # if "bool" not in search_query["query"]:
-            #     search_query["query"]["bool"] = {}
-            # if "must" not in search_query["query"]["bool"]:
-            #     search_query["query"]["bool"] = {"must": []}
-
-            for item in add_bool:
-                search_query["query"]["bool"]["must"].append({"match_phrase": item})
-        if add_top:
-            for item in add_top:
-                search_query["query"]["bool"]["must"].append(item)
-        if add_top_negative:
-            for item in add_top_negative:
-                if "must_not" in search_query["query"]["bool"]:
-                    search_query["query"]["bool"]["must_not"].append(item)
-                else:
-                    search_query["query"]["bool"]["must_not"] = [item]
-        if sort:
-            search_query["sort"] = sort
-
-        if "index" in query_params:
-            index = query_params["index"]
-            if index == "main":
-                index = settings.INDEX_MAIN
-            else:
-                if not request.user.has_perm(f"core.index_{index}"):
-                    message = "Not permitted to search by this index"
-                    message_class = "danger"
-                    return {
-                        "message": message,
-                        "class": message_class,
-                    }
-                if index == "meta":
-                    index = settings.INDEX_META
-                elif index == "internal":
-                    index = settings.INDEX_INT
-                else:
-                    message = "Index is not valid."
-                    message_class = "danger"
-                    return {
-                        "message": message,
-                        "class": message_class,
-                    }
-
-        else:
-            index = settings.INDEX_MAIN
-
-        results = self.query(
-            request.user,  # passed through run_main_query to filter_blacklisted
-            search_query,
-            custom_query=True,
-            index=index,
-            size=size,
-        )
-        if not results:
-            return False
-        if isinstance(results, Exception):
-            message = f"Error: {results.info['error']['root_cause'][0]['type']}"
-            message_class = "danger"
-            return {"message": message, "class": message_class}
-        if len(results["hits"]["hits"]) == 0:
-            message = "No results."
-            message_class = "danger"
-            return {"message": message, "class": message_class}
-
-        results_parsed = parse_results(results)
-
-        if annotate:
-            annotate_results(results_parsed)
-        if "dedup" in query_params:
-            if query_params["dedup"] == "on":
-                dedup = True
-            else:
-                dedup = False
-        else:
-            dedup = False
-
-        if reverse:
-            results_parsed = results_parsed[::-1]
-
-        if dedup:
-            if not dedup_fields:
-                dedup_fields = ["msg", "nick", "ident", "host", "net", "channel"]
-            results_parsed = dedup_list(results_parsed, dedup_fields)
-
-        # if source not in settings.SAFE_SOURCES:
-        #     if settings.ENCRYPTION:
-        #         encrypt_list(request.user, results_parsed, settings.ENCRYPTION_KEY)
-
-        #     if settings.HASHING:
-        #         hash_list(request.user, results_parsed)
-
-        #     if settings.OBFUSCATION:
-        #         obfuscate_list(request.user, results_parsed)
-
-        #     if settings.RANDOMISATION:
-        #         randomise_list(request.user, results_parsed)
-
-        # process_list(results)
-
-        # IMPORTANT! - DO NOT PASS query_params to the user!
-        context = {
-            "object_list": results_parsed,
-            "card": results["hits"]["total"]["value"],
-            "took": results["took"],
-        }
-        if "redacted" in results:
-            context["redacted"] = results["redacted"]
-        if "exemption" in results:
-            context["exemption"] = results["exemption"]
-        if query:
-            context["query"] = query
-        # if settings.DELAY_RESULTS:
-        #     if source not in settings.SAFE_SOURCES:
-        #         if not request.user.has_perm("core.bypass_delay"):
-        #             context["delay"] = settings.DELAY_DURATION
-        # if settings.RANDOMISATION:
-        #     if source not in settings.SAFE_SOURCES:
-        #         if not request.user.has_perm("core.bypass_randomisation"):
-        #             context["randomised"] = True
-        return context
-
-    def query_single_result(self, request, query_params):
-        context = self.query_results(request, query_params, size=100)
-
-        if not context:
-            return {"message": "Failed to run query", "message_class": "danger"}
-        if "message" in context:
-            return context
-        dedup_set = {item["nick"] for item in context["object_list"]}
-        if dedup_set:
-            context["item"] = context["object_list"][0]
-
-        return context

core/db/processing.py

@@ -3,7 +3,7 @@ from datetime import datetime
 from core.lib.threshold import annotate_num_chans, annotate_num_users, annotate_online
 
 
-def annotate_results(results_parsed):
+def annotate_results(results):
     """
     Accept a list of dict objects, search for the number of channels and users.
     Add them to the object.
@@ -11,7 +11,7 @@ def annotate_results(results_parsed):
     """
     # Figure out items with net (not discord)
     nets = set()
-    for x in results_parsed:
+    for x in results:
         if "net" in x:
             nets.add(x["net"])
 
@@ -21,7 +21,7 @@ def annotate_results(results_parsed):
             set(
                 [
                     x["nick"]
-                    for x in results_parsed
+                    for x in results
                     if {"nick", "src", "net"}.issubset(x)
                     and x["src"] == "irc"
                     and x["net"] == net
@@ -32,7 +32,7 @@ def annotate_results(results_parsed):
             set(
                 [
                     x["channel"]
-                    for x in results_parsed
+                    for x in results
                     if {"channel", "src", "net"}.issubset(x)
                     and x["src"] == "irc"
                     and x["net"] == net
@@ -44,7 +44,7 @@ def annotate_results(results_parsed):
         num_users = annotate_num_users(net, channels)
         # Annotate the number channels the user is on
         num_chans = annotate_num_chans(net, nicks)
-        for item in results_parsed:
+        for item in results:
             if "net" in item:
                 if item["net"] == net:
                     if "nick" in item:

core/db/storage.py

@@ -6,10 +6,10 @@ def get_db():
         from core.db.druid import DruidBackend
 
         return DruidBackend()
-    elif settings.DB_BACKEND == "OPENSEARCH":
-        from core.db.opensearch import OpensearchBackend
+    elif settings.DB_BACKEND == "ELASTICSEARCH":
+        from core.db.elastic import ElasticsearchBackend
 
-        return OpensearchBackend()
+        return ElasticsearchBackend()
     elif settings.DB_BACKEND == "MANTICORE":
         from core.db.manticore import ManticoreBackend
 

core/forms.py

@@ -1,9 +1,45 @@
 from django import forms
 from django.contrib.auth.forms import UserCreationForm
+from django.core.exceptions import FieldDoesNotExist
+from django.forms import ModelForm
+
+from core.db import QueryError
+from core.lib.rules import NotificationRuleData
+
+from .models import NotificationRule, NotificationSettings, User
+
+# from django.forms import ModelForm
 
-from .models import User
 
 # Create your forms here.
+class RestrictedFormMixin:
+    """
+    This mixin is used to restrict the queryset of a form to the current user.
+    The request object is passed from the view.
+    Fieldargs is used to pass additional arguments to the queryset filter.
+    """
+
+    fieldargs = {}
+
+    def __init__(self, *args, **kwargs):
+        # self.fieldargs = {}
+        self.request = kwargs.pop("request")
+        super().__init__(*args, **kwargs)
+        for field in self.fields:
+            # Check it's not something like a CharField which has no queryset
+            if not hasattr(self.fields[field], "queryset"):
+                continue
+
+            model = self.fields[field].queryset.model
+            # Check if the model has a user field
+            try:
+                model._meta.get_field("user")
+                # Add the user to the queryset filters
+                self.fields[field].queryset = model.objects.filter(
+                    user=self.request.user, **self.fieldargs.get(field, {})
+                )
+            except FieldDoesNotExist:
+                pass
 
 
 class NewUserForm(UserCreationForm):
@@ -32,3 +68,55 @@ class CustomUserCreationForm(UserCreationForm):
     class Meta:
         model = User
         fields = "__all__"
+
+
+class NotificationSettingsForm(RestrictedFormMixin, ModelForm):
+    class Meta:
+        model = NotificationSettings
+        fields = (
+            "ntfy_topic",
+            "ntfy_url",
+        )
+        help_texts = {
+            "ntfy_topic": "The topic to send notifications to.",
+            "ntfy_url": "Custom NTFY server. Leave blank to use the default server.",
+        }
+
+
+class NotificationRuleForm(RestrictedFormMixin, ModelForm):
+    class Meta:
+        model = NotificationRule
+        fields = (
+            "name",
+            "data",
+            "priority",
+            "topic",
+            "enabled",
+        )
+        help_texts = {
+            "name": "The name of the rule.",
+            "priority": "The priority of the rule.",
+            "topic": "The topic to send notifications to. Leave blank for default.",
+            "enabled": "Whether the rule is enabled.",
+            "data": "The notification rule definition.",
+        }
+
+    def clean(self):
+        cleaned_data = super(NotificationRuleForm, self).clean()
+        data = cleaned_data.get("data")
+        try:
+            parsed_data = NotificationRuleData(self.request.user, data)
+        except ValueError as e:
+            self.add_error("data", f"Parsing error: {e}")
+            return
+        except QueryError as e:
+            self.add_error("data", f"Query error: {e}")
+            return
+
+        # Write back the validated data
+        # We need this to populate the index and source variable if
+        # they are not set
+        to_store = str(parsed_data)
+        cleaned_data["data"] = to_store
+
+        return cleaned_data

core/lib/meta.py

@@ -3,7 +3,7 @@ from math import ceil
 from django.conf import settings
 from numpy import array_split
 
-from core.db.opensearch import client, run_main_query
+from core.db.elastic import client, run_main_query
 
 
 def construct_query(net, nicks):
@@ -48,7 +48,7 @@ def get_meta(request, net, nicks, iter=True):
             request.user,
             query,
             custom_query=True,
-            index=settings.OPENSEARCH_INDEX_META,
+            index=settings.ELASTICSEARCH_INDEX_META,
         )
         if "hits" in results.keys():
             if "hits" in results["hits"]:

core/lib/notify.py

@@ -0,0 +1,42 @@
+import requests
+
+from core.util import logs
+
+NTFY_URL = "https://ntfy.sh"
+
+log = logs.get_logger(__name__)
+
+
+# Actual function to send a message to a topic
+def raw_sendmsg(msg, title=None, priority=None, tags=None, url=None, topic=None):
+    if url is None:
+        url = NTFY_URL
+    headers = {"Title": "Fisk"}
+    if title:
+        headers["Title"] = title
+    if priority:
+        headers["Priority"] = priority
+    if tags:
+        headers["Tags"] = tags
+    try:
+        requests.post(
+            f"{url}/{topic}",
+            data=msg,
+            headers=headers,
+        )
+    except requests.exceptions.ConnectionError as e:
+        log.error(f"Error sending notification: {e}")
+
+
+# Sendmsg helper to send a message to a user's notification settings
+def sendmsg(user, *args, **kwargs):
+    notification_settings = user.get_notification_settings()
+
+    if "topic" not in kwargs:
+        if notification_settings.ntfy_topic is None:
+            # No topic set, so don't send
+            return
+        else:
+            kwargs["topic"] = notification_settings.ntfy_topic
+
+    raw_sendmsg(*args, **kwargs, url=notification_settings.ntfy_url)

core/lib/rules.py

@@ -0,0 +1,151 @@
+from yaml import dump, load
+from yaml.parser import ParserError
+from yaml.scanner import ScannerError
+
+from core.db.storage import db
+from core.models import NotificationRule
+
+try:
+    from yaml import CDumper as Dumper
+    from yaml import CLoader as Loader
+except ImportError:
+    from yaml import Loader, Dumper
+
+from core.lib.notify import sendmsg
+from core.util import logs
+
+log = logs.get_logger("rules")
+
+
+def rule_matched(rule, message, matched):
+    title = f"Rule {rule.name} matched"
+
+    # Dump the message in YAML for readability
+    message = dump(message, Dumper=Dumper, default_flow_style=False)
+    matched = ", ".join([f"{k}: {v}" for k, v in matched.items()])
+
+    notify_message = f"{rule.name} match: {matched}\n{message}"
+    notify_message = notify_message.encode("utf-8", "replace")
+
+    cast = {
+        "title": title,
+        "priority": str(rule.priority),
+    }
+    if rule.topic is not None:
+        cast["topic"] = rule.topic
+    sendmsg(rule.user, notify_message, **cast)
+
+
+def process_rules(data):
+    all_rules = NotificationRule.objects.filter(enabled=True)
+
+    for index, index_messages in data.items():
+        for message in index_messages:
+            for rule in all_rules:
+                parsed_rule = rule.parse()
+                matched = {}
+                if "index" not in parsed_rule:
+                    continue
+                if "source" not in parsed_rule:
+                    continue
+                rule_index = parsed_rule["index"]
+                rule_source = parsed_rule["source"]
+                if not type(rule_index) == list:
+                    rule_index = [rule_index]
+                if not type(rule_source) == list:
+                    rule_source = [rule_source]
+                if index not in rule_index:
+                    continue
+                if message["src"] not in rule_source:
+                    continue
+
+                matched["index"] = index
+                matched["source"] = message["src"]
+
+                rule_field_length = len(parsed_rule.keys())
+                matched_field_number = 0
+                for field, value in parsed_rule.items():
+                    if not type(value) == list:
+                        value = [value]
+                    if field == "src":
+                        continue
+                    if field == "tokens":
+                        for token in value:
+                            if "tokens" in message:
+                                if token in message["tokens"]:
+                                    matched_field_number += 1
+                                    matched[field] = token
+                                    # Break out of the token matching loop
+                                    break
+                        # Continue to next field
+                        continue
+
+                    # Allow partial matches for msg
+                    if field == "msg":
+                        for msg in value:
+                            if "msg" in message:
+                                if msg.lower() in message["msg"].lower():
+                                    matched_field_number += 1
+                                    matched[field] = msg
+                                    # Break out of the msg matching loop
+                                    break
+                        # Continue to next field
+                        continue
+                    if field in message and message[field] in value:
+                        matched_field_number += 1
+                        matched[field] = message[field]
+                if matched_field_number == rule_field_length - 2:
+                    rule_matched(rule, message, matched)
+
+
+class NotificationRuleData(object):
+    def __init__(self, user, data):
+        self.user = user
+        self.data = data
+        self.parsed = None
+
+        self.parse_data()
+        self.validate_permissions()
+
+    def validate_permissions(self):
+        """
+        Validate permissions for the source and index variables.
+        """
+        if "index" in self.parsed:
+            index = self.parsed["index"]
+            if type(index) == list:
+                for i in index:
+                    db.parse_index(self.user, {"index": i}, raise_error=True)
+            else:
+                db.parse_index(self.user, {"index": index}, raise_error=True)
+        else:
+            # Get the default value for the user if not present
+            index = db.parse_index(self.user, {}, raise_error=True)
+            self.parsed["index"] = index
+
+        if "source" in self.parsed:
+            source = self.parsed["source"]
+            if type(source) == list:
+                for i in source:
+                    db.parse_source(self.user, {"source": i}, raise_error=True)
+            else:
+                db.parse_source(self.user, {"source": source}, raise_error=True)
+        else:
+            # Get the default value for the user if not present
+            source = db.parse_source(self.user, {}, raise_error=True)
+            self.parsed["source"] = source
+
+    def parse_data(self):
+        """
+        Parse the data in the text field to YAML.
+        """
+        try:
+            self.parsed = load(self.data, Loader=Loader)
+        except (ScannerError, ParserError) as e:
+            raise ValueError(f"Invalid YAML: {e}")
+
+    def __str__(self):
+        return dump(self.parsed, Dumper=Dumper)
+
+    def get_data(self):
+        return self.parsed

core/management/commands/processing.py

@@ -0,0 +1,24 @@
+import msgpack
+from django.core.management.base import BaseCommand
+from redis import StrictRedis
+
+from core.lib.rules import process_rules
+from core.util import logs
+
+log = logs.get_logger("processing")
+
+
+class Command(BaseCommand):
+    def handle(self, *args, **options):
+        r = StrictRedis(unix_socket_path="/var/run/socks/redis.sock", db=0)
+        p = r.pubsub()
+        p.psubscribe("messages")
+        for message in p.listen():
+            if message:
+                if message["channel"] == b"messages":
+                    data = message["data"]
+                    try:
+                        unpacked = msgpack.unpackb(data, raw=False)
+                    except TypeError:
+                        continue
+                    process_rules(unpacked)

core/migrations/0011_alter_perms_options.py

@@ -0,0 +1,17 @@
+# Generated by Django 4.1.3 on 2022-11-29 12:04
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0010_alter_perms_options'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='perms',
+            options={'permissions': (('bypass_hashing', 'Can bypass field hashing'), ('bypass_blacklist', 'Can bypass the blacklist'), ('bypass_encryption', 'Can bypass field encryption'), ('bypass_obfuscation', 'Can bypass field obfuscation'), ('bypass_delay', 'Can bypass data delay'), ('bypass_randomisation', 'Can bypass data randomisation'), ('post_irc', 'Can post to IRC'), ('post_discord', 'Can post to Discord'), ('query_search', 'Can search with query strings'), ('use_insights', 'Can use the Insights page'), ('index_internal', 'Can use the internal index'), ('index_meta', 'Can use the meta index'), ('index_restricted', 'Can use the restricted index'), ('restricted_sources', 'Can access restricted sources'))},
+        ),
+    ]

core/migrations/0012_notificationrule.py

@@ -0,0 +1,25 @@
+# Generated by Django 4.1.3 on 2023-01-12 15:12
+
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0011_alter_perms_options'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='NotificationRule',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=255)),
+                ('enabled', models.BooleanField(default=True)),
+                ('data', models.TextField()),
+                ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+            ],
+        ),
+    ]

core/migrations/0013_notificationsettings.py

@@ -0,0 +1,24 @@
+# Generated by Django 4.1.3 on 2023-01-12 15:25
+
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0012_notificationrule'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='NotificationSettings',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('ntfy_topic', models.CharField(blank=True, max_length=255, null=True)),
+                ('ntfy_url', models.CharField(blank=True, max_length=255, null=True)),
+                ('user', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+            ],
+        ),
+    ]

core/migrations/0014_notificationrule_priority.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.5 on 2023-01-12 18:06
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0013_notificationsettings'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='notificationrule',
+            name='priority',
+            field=models.IntegerField(choices=[(1, 'min'), (2, 'low'), (3, 'default'), (4, 'high'), (5, 'max')], default=1),
+        ),
+    ]

core/migrations/0015_notificationrule_topic.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.5 on 2023-01-12 18:14
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0014_notificationrule_priority'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='notificationrule',
+            name='topic',
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+    ]

core/models.py

@@ -3,11 +3,26 @@ import logging
 import stripe
 from django.contrib.auth.models import AbstractUser
 from django.db import models
+from yaml import load
+from yaml.parser import ParserError
+from yaml.scanner import ScannerError
 
 from core.lib.customers import get_or_create, update_customer_fields
 
+try:
+    from yaml import CLoader as Loader
+except ImportError:
+    from yaml import Loader
 logger = logging.getLogger(__name__)
 
+PRIORITY_CHOICES = (
+    (1, "min"),
+    (2, "low"),
+    (3, "default"),
+    (4, "high"),
+    (5, "max"),
+)
+
 
 class Plan(models.Model):
     name = models.CharField(max_length=255, unique=True)
@@ -60,6 +75,9 @@ class User(AbstractUser):
         plan_list = [plan.name for plan in self.plans.all()]
         return plan in plan_list
 
+    def get_notification_settings(self):
+        return NotificationSettings.objects.get_or_create(user=self)[0]
+
 
 class Session(models.Model):
     user = models.ForeignKey(User, on_delete=models.CASCADE)
@@ -117,7 +135,36 @@ class Perms(models.Model):
             ("post_discord", "Can post to Discord"),
             ("query_search", "Can search with query strings"),  #
             ("use_insights", "Can use the Insights page"),
-            ("index_int", "Can use the internal index"),
+            ("index_internal", "Can use the internal index"),
             ("index_meta", "Can use the meta index"),
+            ("index_restricted", "Can use the restricted index"),
             ("restricted_sources", "Can access restricted sources"),
         )
+
+
+class NotificationRule(models.Model):
+    user = models.ForeignKey(User, on_delete=models.CASCADE)
+    name = models.CharField(max_length=255)
+    priority = models.IntegerField(choices=PRIORITY_CHOICES, default=1)
+    topic = models.CharField(max_length=255, null=True, blank=True)
+    enabled = models.BooleanField(default=True)
+    data = models.TextField()
+
+    def __str__(self):
+        return f"{self.user} - {self.name}"
+
+    def parse(self):
+        try:
+            parsed = load(self.data, Loader=Loader)
+        except (ScannerError, ParserError) as e:
+            raise ValueError(f"Invalid YAML: {e}")
+        return parsed
+
+
+class NotificationSettings(models.Model):
+    user = models.OneToOneField(User, on_delete=models.CASCADE)
+    ntfy_topic = models.CharField(max_length=255, null=True, blank=True)
+    ntfy_url = models.CharField(max_length=255, null=True, blank=True)
+
+    def __str__(self):
+        return f"Notification settings for {self.user}"

core/static/js/column-shifter.js

@@ -65,11 +65,12 @@ $(document).ready(function(){
                     "file_ext": "off",
                     "file_size": "off",
                     "lang_code": "off",
+                    "tokens": "off",
                     //"lang_name": "off",
-                    "words_noun": "off",
-                    "words_adj": "off",
-                    "words_verb": "off",
-                    "words_adv": "off"
+                    // "words_noun": "off",
+                    // "words_adj": "off",
+                    // "words_verb": "off",
+                    // "words_adv": "off"
                 },
             };
         } else {

core/templates/base.html

@@ -213,6 +213,21 @@
       }
 
     </style>
+    <!-- Piwik --> {# Yes it's in the source, fight me #}
+    <script type="text/javascript">
+      var _paq = _paq || [];
+      _paq.push(['trackPageView']);
+      _paq.push(['enableLinkTracking']);
+      (function() {
+        _paq.push(['setTrackerUrl',  'https://api-a6fe73d3464641fe99ba77e5fdafa19c.s.zm.is']);
+        _paq.push(['setSiteId', 4]);
+        _paq.push(['setApiToken', 'je4TjsrunIM9uD4jrr_DGXJP4_b_Kq6ABhulOLo_Old']);
+        var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0];
+        g.type='text/javascript'; g.async=true; g.defer=true; g.src='https://c87zpt9a74m181wto33r.s.zm.is/embed.js'; s.parentNode.insertBefore(g,s);
+      })();
+    </script>
+    <!-- End Piwik Code -->
+
   </head>
   <body>
 
@@ -234,10 +249,24 @@
           <a class="navbar-item" href="{% url 'home' %}">
             Search
           </a>
+          <a class="navbar-item" href="{% url 'rules' type='page' %}">
+            Rules
+          </a>
           {% if user.is_authenticated %}
+            <div class="navbar-item has-dropdown is-hoverable">
+              <a class="navbar-link">
+                Account
+              </a>
+
+              <div class="navbar-dropdown">
                 <a class="navbar-item" href="{% url 'billing' %}">
                   Billing
                 </a>
+                <a class="navbar-item" href="{% url 'notifications_update' type='page' %}">
+                  Notifications
+                </a>
+              </div>
+            </div>
           {% endif %}
           {% if user.is_superuser %}
             <div class="navbar-item has-dropdown is-hoverable">
@@ -271,15 +300,15 @@
             <div class="buttons">
               {% if not user.is_authenticated %}
                 <a class="button is-info" href="{% url 'signup' %}">
-                  <strong>Sign up</strong>
+                  Sign up
                 </a>
-                <a class="button is-light" href="{% url 'login' %}">
+                <a class="button" href="{% url 'login' %}">
                   Log in
                 </a>
               {% endif %}
 
               {% if user.is_authenticated %}
-                <a class="button is-dark" href="{% url 'logout' %}">Logout</a>
+                <a class="button" href="{% url 'logout' %}">Logout</a>
               {% endif %}
 
             </div>
@@ -320,8 +349,18 @@
     {% endblock %}
     <section class="section">
       <div class="container">
+        {% block content_wrapper %}
           {% block content %}
           {% endblock %}
+        {% endblock %}
+        <div id="modals-here">
+        </div>
+        <div id="windows-here">
+        </div>
+        <div id="widgets-here" style="display: none;">
+          {% block widgets %}
+          {% endblock %}
+        </div>
       </div>
     </section>
   </body>

core/templates/index.html

@@ -1,48 +1,152 @@
-{% extends "base.html" %}
+{% extends 'base.html' %}
 {% load static %}
+{% load joinsep %}
+{% block outer_content %}
+  {% if params.modal == 'context' %}
+    <div
+      style="display: none;"
+      hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
+      hx-post="{% url 'modal_context' %}"
+      hx-vals='{"net": "{{ params.net|escapejs }}",
+      "num": "{{ params.num|escapejs }}",
+      "source": "{{ params.source|escapejs }}",
+      "channel": "{{ params.channel|escapejs }}",
+      "time": "{{ params.time|escapejs }}",
+      "date": "{{ params.date|escapejs }}",
+      "index": "{{ params.index }}",
+      "type": "{{ params.type|escapejs }}",
+      "mtype": "{{ params.mtype|escapejs }}",
+      "nick": "{{ params.nick|escapejs }}"}'
+      hx-target="#modals-here"
+      hx-trigger="load">
+    </div>
+  {% endif %}
+  <script src="{% static 'js/chart.js' %}"></script>
+  <script src="{% static 'tabs.js' %}"></script>
+  <script>
+    function setupTags() {
+      var inputTags = document.getElementById('tags');
+      new BulmaTagsInput(inputTags);
 
-{% block content %}
-  <div class="block">
-    {% for block in blocks %}
-      {% if block.title is not None %}
-        <h1 class="title">{{ block.title }}</h1>
-      {% endif %}
-      <div class="box">
-        <div class="columns">
-          {% if block.column1 is not None %}
-            <div class="column">
-              {{ block.column1 }}
-            </div>
-          {% endif %}
-          {% if block.column2 is not None %}
-            <div class="column">
-              {{ block.column2 }}
-            </div>
-          {% endif %}
-          {% if block.column3 is not None %}
-            <div class="column">
-              {{ block.column3 }}
-            </div>
-          {% endif %}
-        </div>
-        <div class="columns">
-          {% if block.image1 is not None %}
-            <div class="column">
-              <img src="{% static block.image1 %}">
-            </div>
-          {% endif %}
-          {% if block.image2 is not None %}
-            <div class="column">
-              <img src="{% static block.image2 %}">
-            </div>
-          {% endif %}
-          {% if block.image3 is not None %}
-            <div class="column">
-              <img src="{% static block.image3 %}">
-            </div>
-          {% endif %}
+      inputTags.BulmaTagsInput().on('before.add', function(item) {
+        if (item.includes(": ")) {
+          var spl = item.split(": ");
+        } else {
+          var spl = item.split(":");
+        }
+        var field = spl[0];
+        try {
+          var value = JSON.parse(spl[1]);
+        } catch {
+          var value = spl[1];
+        }
+        return `${field}: ${value}`;
+      });
+      inputTags.BulmaTagsInput().on('after.remove', function(item) {
+        var spl = item.split(": ");
+        var field = spl[0];
+        var value = spl[1].trim();
+      });
+    }
+    function populateSearch(field, value) {
+      var inputTags = document.getElementById('tags');
+      inputTags.BulmaTagsInput().add(field+": "+value);
+      //htmx.trigger("#search", "click");
+    }
+  </script>
+
+  <div class="grid-stack" id="grid-stack-main">
+    <div class="grid-stack-item" gs-w="7" gs-h="10" gs-y="0" gs-x="1">
+      <div class="grid-stack-item-content">
+        <nav class="panel">
+          <p class="panel-heading" style="padding: .2em; line-height: .5em;">
+            <i class="fa-solid fa-arrows-up-down-left-right has-text-grey-light"></i>
+            Search
+          </p>
+          <article class="panel-block is-active">
+            {% include 'window-content/search.html' %}
+          </article>
+        </nav>
       </div>
     </div>
-    {% endfor %}
   </div>
+
+  <script>
+    var grid = GridStack.init({
+      cellHeight: 20,
+      cellWidth: 50,
+      cellHeightUnit: 'px',
+      auto: true,
+      float: true,
+      draggable: {handle: '.panel-heading', scroll: false, appendTo: 'body'},
+      removable: false,
+      animate: true,
+    });
+    // GridStack.init();
+    setupTags();
+
+    // a widget is ready to be loaded
+    document.addEventListener('load-widget', function(event) {
+      let container = htmx.find('#widget');
+      // get the scripts, they won't be run on the new element so we need to eval them
+      var scripts = htmx.findAll(container, "script");
+      let widgetelement = container.firstElementChild.cloneNode(true);
+      var new_id = widgetelement.id;
+
+      // check if there's an existing element like the one we want to swap
+      let grid_element = htmx.find('#grid-stack-main');
+      let existing_widget = htmx.find(grid_element, "#"+new_id);
+
+      // get the size and position attributes
+      if (existing_widget) {
+        let attrs = existing_widget.getAttributeNames();
+        for (let i = 0, len = attrs.length; i < len; i++) {
+          if (attrs[i].startsWith('gs-')) { // only target gridstack attributes
+            widgetelement.setAttribute(attrs[i], existing_widget.getAttribute(attrs[i]));
+          }
+        }
+      }
+
+      // clear the queue element
+      container.outerHTML = "";
+
+      // temporary workaround, other widgets can be duplicated, but not results
+      if (widgetelement.id == 'widget-results') {
+        grid.removeWidget("widget-results");
+      }
+
+      grid.addWidget(widgetelement);
+      // re-create the HTMX JS listeners, otherwise HTMX won't work inside the grid
+      htmx.process(widgetelement);
+
+      // update size when the widget is loaded
+      document.addEventListener('load-widget-results', function(evt) {
+        var added_widget = htmx.find(grid_element, '#widget-results');
+        var itemContent = htmx.find(added_widget, ".control");
+        var scrollheight = itemContent.scrollHeight+80;
+        var verticalmargin = 0;
+        var cellheight = grid.opts.cellHeight;
+        var height = Math.ceil((scrollheight + verticalmargin) / (cellheight + verticalmargin));
+        var opts = {
+          h: height,
+        }
+        grid.update(
+          added_widget,
+          opts
+        );
+      });
+
+      // run the JS scripts inside the added element again
+      // for instance, this will fix the dropdown
+      for (var i = 0; i < scripts.length; i++) {
+        eval(scripts[i].innerHTML);
+      }
+    });
+  </script>
+
+{% endblock %}
+{% block widgets %}
+  {% if table or message is not None %}
+    {% include 'partials/results_load.html' %}
+  {% endif %}
 {% endblock %}

core/templates/partials/close-modal.html

@@ -0,0 +1 @@
+<button class="modal-close is-large" aria-label="close"></button>

core/templates/partials/close-widget.html

@@ -0,0 +1,3 @@
+<i
+  class="fa-solid fa-xmark has-text-grey-light float-right"
+  onclick='grid.removeWidget("widget-{{ unique }}");'></i>

core/templates/partials/close-window.html

@@ -0,0 +1,3 @@
+<i
+  class="fa-solid fa-xmark has-text-grey-light float-right"
+  data-script="on click remove the closest <nav/>"></i>

core/templates/partials/results_load.html

@@ -1,20 +1,10 @@
 {% extends 'wm/widget.html' %}
 {% load static %}
 
-{% block widget_options %}
-  gs-w="10" gs-h="1" gs-y="10" gs-x="1"
-{% endblock %}
-
 {% block heading %}
   Results
 {% endblock %}
 
-{% block close_button %}
-  <i
-    class="fa-solid fa-xmark has-text-grey-light float-right"
-    onclick='grid.removeWidget("drilldown-widget-{{ unique }}"); //grid.compact();'></i>
-{% endblock %}
-
 {% block panel_content %}
   {% include 'partials/notify.html' %}
   <script src="{% static 'js/column-shifter.js' %}"></script>
@@ -38,6 +28,6 @@
     {% endif %}
   {% endif %}
 
-  {% include 'ui/drilldown/table_results_partial.html' %}
-  {% include 'ui/drilldown/sentiment_partial.html' %}
+  {% include 'partials/results_table.html' %}
+  {% include 'partials/sentiment_chart.html' %}
 {% endblock %}

core/templates/partials/results_table.html

@@ -4,6 +4,7 @@
 {% load joinsep %}
 {% load urlsafe %}
 {% block table-wrapper %}
+  <script src="{% static 'js/column-shifter.js' %}"></script>
   <div id="drilldown-table" class="column-shifter-container" style="position:relative; z-index:1;">
     {% block table %}
       <div class="nowrap-parent">
@@ -80,11 +81,11 @@
                                 </div>
                                 <div class="nowrap-child">
                                   <a
-                                    hx-get="search/{% querystring table.prefixed_order_by_field=column.order_by_alias.next %}&{{ uri }}"
+                                    hx-get="search/partial/{% querystring table.prefixed_order_by_field=column.order_by_alias.next %}&{{ uri }}"
                                     hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
                                     hx-trigger="click"
-                                    hx-target="#results"
-                                    hx-swap="innerHTML"
+                                    hx-target="#drilldown-table"
+                                    hx-swap="outerHTML"
                                     hx-indicator="#spinner"
                                     style="cursor: pointer;">
                                     {{ column.header }}
@@ -141,14 +142,10 @@
                                   <i class="fa-solid fa-file-slash"></i>
                                 </span>
                               </td>
-                            {% elif column.name == 'tokens' %}
-                              <td class="{{ column.name }} wrap" style="max-width: 10em">
-                                {{ cell|joinsep:',' }}
-                              </td>
                             {% elif column.name == 'src' %}
                               <td class="{{ column.name }}">
                                 <a
-                                  class="has-text-link is-underlined"
+                                  class="has-text-grey"
                                   onclick="populateSearch('src', '{{ cell|escapejs }}')">
                                   {% if row.cells.src == 'irc' %}
                                     <span class="icon" data-tooltip="IRC">
@@ -173,7 +170,7 @@
                             {% elif column.name == 'type' or column.name == 'mtype' %}
                               <td class="{{ column.name }}">
                                 <a
-                                  class="has-text-link is-underlined"
+                                  class="has-text-grey"
                                   onclick="populateSearch('{{ column.name }}', '{{ cell|escapejs }}')">
                                   {% if cell == 'msg' %}
                                     <span class="icon" data-tooltip="Message">
@@ -281,7 +278,7 @@
                                       </span>
                                     {% endif %}
                                   </div>
-                                  <a class="nowrap-child has-text-link is-underlined" onclick="populateSearch('nick', '{{ cell|escapejs }}')">
+                                  <a class="nowrap-child has-text-grey" onclick="populateSearch('nick', '{{ cell|escapejs }}')">
                                     {{ cell }}
                                   </a>
                                   <div class="nowrap-child">
@@ -301,7 +298,7 @@
                                         hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
                                         hx-post="{% url 'modal_drilldown' type='window' %}"
                                         hx-vals='{"net": "{{ row.cells.net }}", "nick": "{{ row.cells.nick }}", "channel": "{{ row.cells.channel }}"}'
-                                        hx-target="#items-here"
+                                        hx-target="#windows-here"
                                         hx-swap="afterend"
                                         hx-trigger="click"
                                         class="has-text-black">
@@ -336,7 +333,7 @@
                                 {% if cell != '—' %}
                                   <div class="nowrap-parent">
                                     <a
-                                      class="nowrap-child has-text-link is-underlined"
+                                      class="nowrap-child has-text-grey"
                                       onclick="populateSearch('channel', '{{ cell|escapejs }}')">
                                       {{ cell }}
                                     </a>
@@ -364,30 +361,22 @@
                                   </span>
                                 {% endif %}
                               </td>
-                            {% elif column.name|slice:":6" == "words_" %}
+                            {% elif column.name == "tokens" %}
                               <td class="{{ column.name }}">
-                                {% if cell.0.1|length == 0 %}
-                                  <a
-                                    class="tag is-info"
-                                    onclick="populateSearch('{{ column.name }}', '{{ cell }}')">
-                                    {{ cell }}
-                                  </a>
-                                {% else %}
                                 <div class="tags">
                                   {% for word in cell %}
                                     <a
-                                        class="tag is-info"
+                                      class="tag"
                                       onclick="populateSearch('{{ column.name }}', '{{ word }}')">
                                       {{ word }}
                                     </a>
                                   {% endfor %}
                                 </div>
-                                {% endif %}
                               </td>
                             {% else %}
                               <td class="{{ column.name }}">
                                 <a
-                                  class="has-text-link is-underlined"
+                                  class="has-text-grey"
                                   onclick="populateSearch('{{ column.name }}', '{{ cell|escapejs }}')">
                                   {{ cell }}
                                 </a>
@@ -433,11 +422,11 @@
             <a
               class="pagination-previous is-flex-grow-0 {% if not table.page.has_previous %}is-hidden-mobile{% endif %}"
               {% if table.page.has_previous %}
-                hx-get="search/{% querystring table.prefixed_page_field=table.page.previous_page_number %}&{{ uri }}"
+                hx-get="search/partial/{% querystring table.prefixed_page_field=table.page.previous_page_number %}&{{ uri }}"
                 hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
                 hx-trigger="click"
-                hx-target="#results"
-                hx-swap="innerHTML"
+                hx-target="#drilldown-table"
+                hx-swap="outerHTML"
                 hx-indicator="#spinner"
               {% else %}
                 href="#"
@@ -453,11 +442,11 @@
             <a
               class="pagination-next is-flex-grow-0 {% if not table.page.has_next %}is-hidden-mobile{% endif %}"
               {% if table.page.has_next %}
-                hx-get="search/{% querystring table.prefixed_page_field=table.page.next_page_number %}&{{ uri }}"
+                hx-get="search/partial/{% querystring table.prefixed_page_field=table.page.next_page_number %}&{{ uri }}"
                 hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
                 hx-trigger="click"
-                hx-target="#results"
-                hx-swap="innerHTML"
+                hx-target="#drilldown-table"
+                hx-swap="outerHTML"
                 hx-indicator="#spinner"
               {% else %}
                 href="#"
@@ -482,11 +471,11 @@
                       {% if p == table.page.number %}
                         href="#"
                       {% else %}
-                        hx-get="search/{% querystring table.prefixed_page_field=p %}&{{ uri }}"
+                        hx-get="search/partial/{% querystring table.prefixed_page_field=p %}&{{ uri }}"
                         hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
                         hx-trigger="click"
-                        hx-target="#results"
-                        hx-swap="innerHTML"
+                        hx-target="#drilldown-table"
+                        hx-swap="outerHTML"
                         hx-indicator="#spinner"
                       {% endif %}
                     >

core/templates/partials/rule-list.html

@@ -0,0 +1,73 @@
+{% include 'partials/notify.html' %}
+
+<table
+  class="table is-fullwidth is-hoverable"
+  hx-target="#{{ context_object_name }}-table"
+  id="{{ context_object_name }}-table"
+  hx-swap="outerHTML"
+  hx-trigger="{{ context_object_name_singular }}Event from:body"
+  hx-get="{{ list_url }}">
+  <thead>
+    <th>id</th>
+    <th>user</th>
+    <th>name</th>
+    <th>priority</th>
+    <th>topic</th>
+    <th>enabled</th>
+    <th>data length</th>
+    <th>actions</th>
+  </thead>
+  {% for item in object_list %}
+    <tr>
+      <td>{{ item.id }}</td>
+      <td>{{ item.user }}</td>
+      <td>{{ item.name }}</td>
+      <td>{{ item.priority }}</td>
+      <td>{{ item.topic|default_if_none:"—" }}</td>
+      <td>
+        {% if item.enabled %}
+          <span class="icon">
+            <i class="fa-solid fa-check"></i>
+          </span>
+        {% else %}
+          <span class="icon">
+            <i class="fa-solid fa-xmark"></i>
+          </span>
+        {% endif %}
+      </td>
+      <td>{{ item.data|length }}</td>
+      <td>
+        <div class="buttons">
+          <button
+            hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
+            hx-get="{% url 'rule_update' type=type pk=item.id %}"
+            hx-trigger="click"
+            hx-target="#{{ type }}s-here"
+            hx-swap="innerHTML"
+            class="button">
+            <span class="icon-text">
+              <span class="icon">
+                <i class="fa-solid fa-pencil"></i>
+              </span>
+            </span>
+          </button>
+          <button
+            hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
+            hx-delete="{% url 'rule_delete' type=type pk=item.id %}"
+            hx-trigger="click"
+            hx-target="#modals-here"
+            hx-swap="innerHTML"
+            hx-confirm="Are you sure you wish to delete {{ item.name }}?"
+            class="button">
+            <span class="icon-text">
+              <span class="icon">
+                <i class="fa-solid fa-xmark"></i>
+              </span>
+            </span>
+          </button>
+        </div>
+      </td>
+    </tr>
+  {% endfor %}
+
+</table>

core/templates/ui/drilldown/drilldown.html

@@ -1,163 +0,0 @@
-{% extends "base.html" %}
-{% load static %}
-{% load joinsep %}
-{% block outer_content %}
-  {% if params.modal == 'context' %}
-    <div
-      style="display: none;"
-      hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
-      hx-post="{% url 'modal_context' %}"
-      hx-vals='{"net": "{{ params.net|escapejs }}",
-      "num": "{{ params.num|escapejs }}",
-      "source": "{{ params.source|escapejs }}",
-      "channel": "{{ params.channel|escapejs }}",
-      "time": "{{ params.time|escapejs }}",
-      "date": "{{ params.date|escapejs }}",
-      "index": "{{ params.index }}",
-      "type": "{{ params.type|escapejs }}",
-      "mtype": "{{ params.mtype|escapejs }}",
-      "nick": "{{ params.nick|escapejs }}"}'
-      hx-target="#modals-here"
-      hx-trigger="load">
-    </div>
-  {% endif %}
-  <script src="{% static 'js/chart.js' %}"></script>
-  <script src="{% static 'tabs.js' %}"></script>
-  <script>
-    function setupTags() {
-      var inputTags = document.getElementById('tags');
-      new BulmaTagsInput(inputTags);
-
-      inputTags.BulmaTagsInput().on('before.add', function(item) {
-        if (item.includes(": ")) {
-          var spl = item.split(": ");
-        } else {
-          var spl = item.split(":");
-        }
-        var field = spl[0];
-        try {
-          var value = JSON.parse(spl[1]);
-        } catch {
-          var value = spl[1];
-        }
-        return `${field}: ${value}`;
-      });
-      inputTags.BulmaTagsInput().on('after.remove', function(item) {
-        var spl = item.split(": ");
-        var field = spl[0];
-        var value = spl[1].trim();
-      });
-    }
-    function populateSearch(field, value) {
-      var inputTags = document.getElementById('tags');
-      inputTags.BulmaTagsInput().add(field+": "+value);
-      //htmx.trigger("#search", "click");
-    }
-  </script>
-
-  <div class="grid-stack" id="grid-stack-main">
-    <div class="grid-stack-item" gs-w="7" gs-h="10" gs-y="0" gs-x="1">
-      <div class="grid-stack-item-content">
-        <nav class="panel">
-          <p class="panel-heading" style="padding: .2em; line-height: .5em;">
-            <i class="fa-solid fa-arrows-up-down-left-right has-text-grey-light"></i>
-            Search
-          </p>
-          <article class="panel-block is-active">
-            {% include 'ui/drilldown/search_partial.html' %}
-          </article>
-        </nav>
-      </div>
-    </div>
-  </div>
-
-  <script>
-    var grid = GridStack.init({
-      cellHeight: 20,
-      cellWidth: 50,
-      cellHeightUnit: 'px',
-      auto: true,
-      float: true,
-      draggable: {handle: '.panel-heading', scroll: false, appendTo: 'body'},
-      removable: false,
-      animate: true,
-    });
-    // GridStack.init();
-    setupTags();
-
-    // a widget is ready to be loaded
-    document.addEventListener('load-widget', function(event) {
-      let container = htmx.find('#drilldown-widget');
-      // get the scripts, they won't be run on the new element so we need to eval them
-      var scripts = htmx.findAll(container, "script");
-      let widgetelement = container.firstElementChild.cloneNode(true);
-
-      // check if there's an existing element like the one we want to swap
-      let grid_element = htmx.find('#grid-stack-main');
-      let existing_widget = htmx.find(grid_element, '#drilldown-widget-results');
-
-      // get the size and position attributes
-      if (existing_widget) {
-        let attrs = existing_widget.getAttributeNames();
-        for (let i = 0, len = attrs.length; i < len; i++) {
-          if (attrs[i].startsWith('gs-')) { // only target gridstack attributes
-            widgetelement.setAttribute(attrs[i], existing_widget.getAttribute(attrs[i]));
-          }
-        }
-      }
-
-      // clear the queue element
-      container.outerHTML = "";
-
-      // temporary workaround, other widgets can be duplicated, but not results
-      if (widgetelement.id == 'drilldown-widget-results') {
-        grid.removeWidget("drilldown-widget-{{ unique }}");
-      }
-
-      grid.addWidget(widgetelement);
-      // re-create the HTMX JS listeners, otherwise HTMX won't work inside the grid
-      htmx.process(widgetelement);
-
-      // update size when the widget is loaded
-      document.addEventListener('load-widget-results', function(evt) {
-        var added_widget = htmx.find(grid_element, '#drilldown-widget-results');
-        console.log(added_widget);
-        var itemContent = htmx.find(added_widget, ".control");
-        console.log(itemContent);
-        var scrollheight = itemContent.scrollHeight+80;
-        var verticalmargin = 0;
-        var cellheight = grid.opts.cellHeight;
-        var height = Math.ceil((scrollheight + verticalmargin) / (cellheight + verticalmargin));
-        var opts = {
-          h: height,
-        }
-        grid.update(
-          added_widget,
-          opts
-        );
-      });
-
-      // run the JS scripts inside the added element again
-      // for instance, this will fix the dropdown
-      for (var i = 0; i < scripts.length; i++) {
-        eval(scripts[i].innerHTML);
-      }
-    });
-  </script>
-
-  <div id="modals-here">
-  </div>
-  <div id="items-here">
-  </div>
-  <div id="widgets-here" style="display: none;">
-  </div>
-  <div id="results" style="display: none;">
-    {% if table %}
-      {% include 'widgets/table_results.html' %}
-    {% endif %}
-  </div>
-  <script>
-
-  </script>
-
-{% endblock %}

core/templates/window-content/context.html

@@ -1,122 +0,0 @@
-{% load index %}
-{% load static %}
-
-<script src="{% static 'modal.js' %}"></script>
-<script>
-  document.addEventListener("restore-modal-scroll", function(event) {
-    var modalContent = document.getElementsByClassName("modal-content")[0];
-    var maxScroll = modalContent.scrollHeight - modalContent.offsetHeight;
-    var scrollpos = localStorage.getItem('scrollpos_modal_content');
-    if (scrollpos == 'BOTTOM') {
-      modalContent.scrollTop = maxScroll;
-    } else if (scrollpos) {
-      modalContent.scrollTop = scrollpos;
-    };
-  });
-
-  document.addEventListener("htmx:beforeSwap", function(event) {
-    var modalContent = document.getElementsByClassName("modal-content")[0];
-    var scrollpos = modalContent.scrollTop;
-    if(modalContent.scrollTop === (modalContent.scrollHeight - modalContent.offsetHeight)) {
-      localStorage.setItem('scrollpos_modal_content', 'BOTTOM');
-    } else {
-      localStorage.setItem('scrollpos_modal_content', scrollpos);
-    }
-
-  });
-</script>
-<style>
-  #tab-content-{{ unique }} div {
-    display: none;
-  }
-
-  #tab-content-{{ unique }} div.is-active {
-    display: block;
-  }
-</style>
-
-<div id="modal" class="modal is-active is-clipped">
-  <div class="modal-background"></div>
-  <div class="modal-content">
-    <div class="box">
-      {% include 'partials/notify.html' %}
-      <div class="tabs is-toggle is-fullwidth is-info" id="tabs-{{ unique }}">
-        <ul>
-          <li class="is-active" data-tab="1">
-            <a>
-              <span class="icon is-small"><i class="fa-solid fa-message-arrow-down"></i></span>
-              <span>Scrollback</span>
-            </a>
-          </li>
-          <li data-tab="2">
-            <a>
-              <span class="icon is-small"><i class="fa-solid fa-messages"></i></span>
-              <span>Context</span>
-            </a>
-          </li>
-          <li data-tab="3">
-            <a>
-              <span class="icon is-small"><i class="fa-solid fa-message"></i></span>
-              <span>Message</span>
-            </a>
-          </li>
-          <li data-tab="4">
-            <a>
-              <span class="icon is-small"><i class="fa-solid fa-asterisk"></i></span>
-              <span>Info</span>
-            </a>
-          </li>
-        </ul>
-      </div>
-      <div id="tab-content-{{ unique }}">
-        <div class="is-active" data-content="1">
-          <h4 class="subtitle is-4">Scrollback of {{ channel }} on {{ net }}{{ num }}</h4>
-          {% include 'modals/context_table.html' %}
-          {% if user.is_superuser and src == 'irc' %}
-            <form method="PUT">
-              <article class="field has-addons">
-                <article class="control is-expanded has-icons-left">
-                  <input id="context-input" name="msg" class="input" type="text" placeholder="Type your message here">
-                  <span class="icon is-small is-left">
-                    <i class="fas fa-magnifying-glass"></i>
-                  </span>
-                </article>
-                <article class="control">
-                  <article class="field">
-                    <button
-                      id="search"
-                      class="button is-info is-fullwidth"
-                      hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
-                      hx-put="{% url 'threshold_irc_msg' net num %}"
-                      hx-vals='{"channel": "{{ channel }}", "nick": "{{ nick }}"}'
-                      hx-trigger="click"
-                      hx-target="#context-input"
-                      hx-swap="outerHTML">
-                      Send
-                    </button>
-                  </article>
-                </article>
-              </article>
-            </form>
-          {% endif %}
-
-        </div>
-        <div data-content="2">
-          <h4 class="subtitle is-4">Scrollback of {{ channel }} on {{ net }}{{ num }} around {{ ts }}</h4>
-          Context
-        </div>
-        <div data-content="3">
-          <h4 class="subtitle is-4">Message details</h4>
-          Message deetails
-        </div>
-        <div data-content="4">
-          <h4 class="subtitle is-4">Information about {{ channel }} on {{ net }}{{ num }}</h4>
-          info
-        </div>
-      </div>
-    </div>
-    <script>initTabs("{{ unique }}");</script>
-    <button class="modal-close is-large" aria-label="close"></button>
-  </div>
-</div>
-

core/templates/window-content/context_table.html

@@ -1,177 +0,0 @@
-<article class="table-container" id="modal-context-table">
-  <table class="table is-fullwidth">
-    <thead>
-      <th></th>
-      <th></th>
-      <th></th>
-    </thead>
-    <tbody>
-      {% for item in object_list %}
-        {% if item.type == 'control' %}
-          <tr>
-            <td></td>
-            <td>
-              <span class="icon has-text-grey" data-tooltip="Hidden">
-                <i class="fa-solid fa-file-slash"></i>
-              </span>
-            </td>
-            <td>
-              <p class="has-text-grey">Hidden {{ item.hidden }} similar result{% if item.hidden > 1%}s{% endif %}</p>
-            </td>
-          </tr>
-        {% else %}
-          <tr>
-            <td>{{ item.time }}</td>
-            <td>
-              {% if item.type != 'znc' and item.type != 'self' and query is not True %}
-                <article class="nowrap-parent">
-                  <article class="nowrap-child">
-                    {% if item.type == 'msg' %}
-                      <span class="icon" data-tooltip="Message">
-                        <i class="fa-solid fa-message"></i>
-                      </span>
-                    {% elif item.type == 'join' %}
-                      <span class="icon" data-tooltip="Join">
-                        <i class="fa-solid fa-person-to-portal"></i>
-                      </span>
-                    {% elif item.type == 'part' %}
-                      <span class="icon" data-tooltip="Part">
-                        <i class="fa-solid fa-person-from-portal"></i>
-                      </span>
-                    {% elif item.type == 'quit' %}
-                      <span class="icon" data-tooltip="Quit">
-                        <i class="fa-solid fa-circle-xmark"></i>
-                      </span>
-                    {% elif item.type == 'kick' %}
-                      <span class="icon" data-tooltip="Kick">
-                        <i class="fa-solid fa-user-slash"></i>
-                      </span>
-                    {% elif item.type == 'nick' %}
-                      <span class="icon" data-tooltip="Nick">
-                        <i class="fa-solid fa-signature"></i>
-                      </span>
-                    {% elif item.type == 'mode' %}
-                      <span class="icon" data-tooltip="Mode">
-                        <i class="fa-solid fa-gear"></i>
-                      </span>
-                    {% elif item.type == 'action' %}
-                      <span class="icon" data-tooltip="Action">
-                        <i class="fa-solid fa-exclamation"></i>
-                      </span>
-                    {% elif item.type == 'notice' %}
-                      <span class="icon" data-tooltip="Notice">
-                        <i class="fa-solid fa-message-code"></i>
-                      </span>
-                    {% elif item.type == 'conn' %}
-                      <span class="icon" data-tooltip="Connection">
-                        <i class="fa-solid fa-cloud-exclamation"></i>
-                      </span>
-                    {% elif item.type == 'znc' %}
-                      <span class="icon" data-tooltip="ZNC">
-                        <i class="fa-brands fa-unity"></i>
-                      </span>
-                    {% elif item.type == 'query' %}
-                      <span class="icon" data-tooltip="Query">
-                        <i class="fa-solid fa-message"></i>
-                      </span>
-                    {% elif item.type == 'highlight' %}
-                      <span class="icon" data-tooltip="Highlight">
-                        <i class="fa-solid fa-exclamation"></i>
-                      </span>
-                    {% elif item.type == 'who' %}
-                      <span class="icon" data-tooltip="Who">
-                        <i class="fa-solid fa-passport"></i>
-                      </span>
-                    {% elif item.type == 'topic' %}
-                      <span class="icon" data-tooltip="Topic">
-                        <i class="fa-solid fa-sign"></i>
-                      </span>
-                    {% else %}
-                      {{ item.type }}
-                    {% endif %}
-                    {% if item.online is True %}
-                      <span class="icon has-text-success has-tooltip-success" data-tooltip="Online">
-                        <i class="fa-solid fa-circle"></i>
-                      </span>
-                    {% elif item.online is False %}
-                      <span class="icon has-text-danger has-tooltip-danger" data-tooltip="Offline">
-                        <i class="fa-solid fa-circle"></i>
-                      </span>
-                    {% else %}
-                      <span class="icon has-text-warning has-tooltip-warning" data-tooltip="Unknown">
-                        <i class="fa-solid fa-circle"></i>
-                      </span>
-                    {% endif %}
-                    {% if item.src == 'irc' %}
-                      <a
-                        hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
-                        hx-post="{% url 'modal_drilldown' %}"
-                        hx-vals='{"net": "{{ item.net|escapejs }}", "nick": "{{ item.nick|escapejs }}", "channel": "{{ item.channel|escapejs }}"}'
-                        hx-target="#modals-here"
-                        hx-trigger="click"
-                        class="has-text-black">
-                        <span class="icon" data-tooltip="Open drilldown modal">
-                          <i class="fa-solid fa-album"></i>
-                        </span>
-                      </a>
-                    {% endif %}
-                  </article>
-                  <a class="nowrap-child has-text-link is-underlined" onclick="populateSearch('nick', '{{ item.nick|escapejs }}')">
-                    {{ item.nick }}
-                  </a>
-                  {% if item.num_chans != '—' %}
-                    <article class="nowrap-child">
-                      <span class="tag">
-                        {{ item.num_chans }}
-                      </span>
-                    </article>
-                  {% endif %}
-                </article>
-              {% endif %}
-              {% if item.type == 'self' %}
-                <span class="icon has-text-primary" data-tooltip="You">
-                  <i class="fa-solid fa-message-check"></i>
-                </span>
-              {% elif item.type == 'znc' %}
-                <span class="icon has-text-info" data-tooltip="ZNC">
-                  <i class="fa-brands fa-unity"></i>
-                </span>
-              {% elif query %}
-                <span class="icon has-text-info" data-tooltip="Auth">
-                  <i class="fa-solid fa-passport"></i>
-                </span>
-              {% endif %}
-            </td>
-            <td class="wrap">{{ item.msg }}</td>
-          </tr>
-        {% endif %}
-      {% endfor %}
-    </tbody>
-  </table>
-  {% if object_list %}
-    <div
-      class="modal-refresh"
-      style="display: none;"
-      hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
-      hx-post="{% url 'modal_context_table' %}"
-      hx-vals='{"net": "{{ net }}",
-      "num": "{{ num }}",
-      "src": "{{ src }}",
-      "channel": "{{ channel }}",
-      "time": "{{ time }}",
-      "date": "{{ date }}",
-      "index": "{{ index }}",
-      "type": "{{ type }}",
-      "mtype": "{{ mtype }}",
-      "nick": "{{ nick }}",
-      "dedup": "{{ params.dedup }}"}'
-      hx-target="#modal-context-table"
-      hx-trigger="every 5s">
-    </div>
-  {% endif %}
-</article>
-
-<script>
-  var modal_event = new Event('restore-modal-scroll');
-  document.dispatchEvent(modal_event);
-</script>

core/templates/window-content/object-form.html

@@ -0,0 +1,34 @@
+{% include 'partials/notify.html' %}
+{% if page_title is not None %}
+  <h1 class="title is-4">{{ page_title }}</h1>
+{% endif %}
+{% if page_subtitle is not None %}
+  <h1 class="subtitle">{{ page_subtitle }}</h1>
+{% endif %}
+{% load crispy_forms_tags %}
+
+{% load crispy_forms_bulma_field %}
+<form
+  hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
+  hx-post="{{ submit_url }}"
+  hx-target="#modals-here"
+  hx-swap="innerHTML">
+  {% csrf_token %}
+  {{ form|crispy }}
+  {% if hide_cancel is not True %}
+    <button
+      type="button"
+      class="button is-light modal-close-button">
+      Cancel
+    </button>
+  {% endif %}
+  <button type="submit" class="button modal-close-button">Submit</button>
+</form>
+
+
+
+
+
+
+
+

core/templates/window-content/object.html

@@ -0,0 +1,45 @@
+{% include 'partials/notify.html' %}
+{% if page_title is not None %}
+  <h1 class="title is-4">{{ page_title }}</h1>
+{% endif %}
+{% if page_subtitle is not None %}
+  <h1 class="subtitle">{{ page_subtitle }}</h1>
+{% endif %}
+<div class="buttons">
+
+  {% if submit_url is not None %}
+    <button
+      hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
+      hx-get="{{ submit_url }}"
+      hx-trigger="click"
+      hx-target="#modals-here"
+      class="button">
+      <span class="icon-text">
+        <span class="icon">
+          <i class="fa-solid fa-plus"></i>
+        </span>
+        <span>{{ title_singular }}</span>
+      </span>
+    </button>
+  {% endif %}
+  {% if delete_all_url is not None %}
+    <button
+      hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
+      hx-delete="{{ delete_all_url }}"
+      hx-trigger="click"
+      hx-target="#modals-here"
+      hx-swap="innerHTML"
+      hx-confirm="Are you sure you wish to delete all {{ context_object_name }}?"
+      class="button">
+      <span class="icon-text">
+        <span class="icon">
+          <i class="fa-solid fa-xmark"></i>
+        </span>
+        <span>Delete all {{ context_object_name }} </span>
+      </span>
+    </button>
+  {% endif %}
+</div>
+
+{% include detail_template %}
+

core/templates/window-content/objects.html

@@ -0,0 +1,45 @@
+{% include 'partials/notify.html' %}
+{% if page_title is not None %}
+  <h1 class="title is-4">{{ page_title }}</h1>
+{% endif %}
+{% if page_subtitle is not None %}
+  <h1 class="subtitle">{{ page_subtitle }}</h1>
+{% endif %}
+<div class="buttons">
+
+  {% if submit_url is not None %}
+    <button
+      hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
+      hx-get="{{ submit_url }}"
+      hx-trigger="click"
+      hx-target="#modals-here"
+      class="button">
+      <span class="icon-text">
+        <span class="icon">
+          <i class="fa-solid fa-plus"></i>
+        </span>
+        <span>{{ title_singular }}</span>
+      </span>
+    </button>
+  {% endif %}
+  {% if delete_all_url is not None %}
+    <button
+      hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
+      hx-delete="{{ delete_all_url }}"
+      hx-trigger="click"
+      hx-target="#modals-here"
+      hx-swap="innerHTML"
+      hx-confirm="Are you sure you wish to delete all {{ context_object_name }}?"
+      class="button">
+      <span class="icon-text">
+        <span class="icon">
+          <i class="fa-solid fa-xmark"></i>
+        </span>
+        <span>Delete all {{ context_object_name }} </span>
+      </span>
+    </button>
+  {% endif %}
+</div>
+
+{% include list_template %}
+

core/templates/window-content/results.html

@@ -0,0 +1,25 @@
+{% load static %}
+
+{% include 'partials/notify.html' %}
+{% if cache is not None %}
+  <span class="icon has-tooltip-bottom" data-tooltip="Cached">
+    <i class="fa-solid fa-database"></i>
+  </span>
+{% endif %}
+
+fetched {{ table.data|length }} hits in {{ took }}ms
+
+{% if exemption is not None %}
+  <span class="icon has-tooltip-bottom" data-tooltip="God mode">
+    <i class="fa-solid fa-book-bible"></i>
+  </span>
+{% else %}
+  {% if redacted is not None %}
+    <span class="icon has-tooltip-bottom" data-tooltip="{{ redacted }} redacted">
+      <i class="fa-solid fa-mask"></i>
+    </span>
+  {% endif %}
+{% endif %}
+
+{% include 'partials/results_table.html' %}
+{% include 'partials/sentiment_chart.html' %}

core/templates/window-content/search.html

@@ -1,6 +1,6 @@
 <form class="skipEmptyFields" method="POST" hx-post="{% url 'search' %}"
   hx-trigger="change"
-  hx-target="#results"
+  hx-target="#widgets-here"
   hx-swap="innerHTML"
   hx-indicator="#spinner">
   {% csrf_token %}
@@ -11,7 +11,7 @@
           <input
             hx-post="{% url 'search' %}"
             hx-trigger="keyup changed delay:200ms"
-            hx-target="#results"
+            hx-target="#widgets-here"
             hx-swap="innerHTML"
             name="query"
             value="{{ params.query }}"
@@ -26,10 +26,10 @@
           <div class="field">
             <button
               id="search"
-              class="button is-info is-fullwidth"
+              class="button is-fullwidth"
               hx-post="{% url 'search' %}"
               hx-trigger="click"
-              hx-target="#results"
+              hx-target="#widgets-here"
               hx-swap="innerHTML">
               Search
             </button>
@@ -41,7 +41,7 @@
       <div class="nowrap-parent">
         <div
           data-script="on click toggle .is-hidden on #options"
-          class="button is-light has-text-link is-right nowrap-child">
+          class="button is-right nowrap-child">
           Options
         </div>
         <div class="nowrap-child">
@@ -394,7 +394,7 @@
     <input
       hx-trigger="change"
       hx-post="{% url 'search' %}"
-      hx-target="#results"
+      hx-target="#widgets-here"
       hx-swap="innerHTML"
       id="tags"
       class="input"

core/templates/windows/drilldown.html

@@ -1,4 +1,4 @@
-{% extends 'wm/magnet.html' %}
+{% extends 'wm/window.html' %}
 
 {% block heading %}
   Drilldown

core/templates/wm/modal.html

@@ -12,8 +12,9 @@
   <div class="modal-content">
     <div class="box">
       {% block modal_content %}
+        {% include window_content %}
       {% endblock %}
-      <button class="modal-close is-large" aria-label="close"></button>
+      {% include 'partials/close-modal.html' %}
     </div>
   </div>
 </div>

core/templates/wm/page.html

@@ -0,0 +1,6 @@
+{% extends "base.html" %}
+
+
+{% block content %}
+  {% include window_content %}
+{% endblock %}

core/templates/wm/panel.html

@@ -3,9 +3,7 @@
   <p class="panel-heading" style="padding: .2em; line-height: .5em;">
     <i class="fa-solid fa-arrows-up-down-left-right has-text-grey-light"></i>
     {% block close_button %}
-      <i
-        class="fa-solid fa-xmark has-text-grey-light float-right"
-        data-script="on click remove the closest <nav/>"></i>
+      {% include 'partials/close-window.html' %}
     {% endblock %}
     {% block heading %}
     {% endblock %}

core/templates/wm/widget.html

@@ -1,24 +1,24 @@
-<div id="drilldown-widget">
-  <div id="drilldown-widget-{{ unique }}" class="grid-stack-item" {% block widget_options %}{% endblock %}>
+<div id="widget">
+  <div id="widget-{{ unique }}" class="grid-stack-item" {% block widget_options %}gs-w="10" gs-h="1" gs-y="10" gs-x="1"{% endblock %}>
     <div class="grid-stack-item-content">
 
       <nav class="panel">
         <p class="panel-heading" style="padding: .2em; line-height: .5em;">
           <i class="fa-solid fa-arrows-up-down-left-right has-text-grey-light"></i>
           {% block close_button %}
-            <i
-              class="fa-solid fa-xmark has-text-grey-light float-right"
-              onclick='grid.removeWidget("drilldown-widget-{{ unique }}");'></i>
+            {% include 'partials/close-widget.html' %}
           {% endblock %}
           <i
             class="fa-solid fa-arrows-minimize has-text-grey-light float-right"
             onclick='grid.compact();'></i>
           {% block heading %}
+            {{ title }}
           {% endblock %}
         </p>
         <article class="panel-block is-active">
           <div class="control">
             {% block panel_content %}
+              {% include window_content %}
             {% endblock %}
           </div>
         </article>

core/templates/wm/window.html

@@ -1,8 +1,10 @@
 <magnet-block attract-distance="10" align-to="outer|center" class="floating-window">
   {% extends 'wm/panel.html' %}
   {% block heading %}
+    {{ title }}
   {% endblock %}
 
   {% block panel_content %}
+    {% include window_content %}
   {% endblock %}
 </magnet-block>

core/views/helpers.py

@@ -8,56 +8,485 @@
 # from siphashc import siphash
 # from sortedcontainers import SortedSet
 
+import uuid
+
 # from core import r
-from django.conf import settings
+from django.core.exceptions import ImproperlyConfigured
+from django.core.paginator import Paginator
+from django.db.models import QuerySet
+from django.http import Http404, HttpResponse, HttpResponseBadRequest
+from django.urls import reverse
+from django.views.generic.detail import DetailView
+from django.views.generic.edit import CreateView, DeleteView, UpdateView
+from django.views.generic.list import ListView
+from rest_framework.parsers import FormParser
+
+from core.util import logs
+
+log = logs.get_logger(__name__)
 
 
-class SearchDenied:
-    def __init__(self, key, value):
-        self.key = key
-        self.value = value
-
-
-class LookupDenied:
-    def __init__(self, key, value):
-        self.key = key
-        self.value = value
-
-
-def remove_defaults(query_params):
-    for field, value in list(query_params.items()):
-        if field in settings.DRILLDOWN_DEFAULT_PARAMS:
-            if value == settings.DRILLDOWN_DEFAULT_PARAMS[field]:
-                del query_params[field]
-
-
-def add_defaults(query_params):
-    for field, value in settings.DRILLDOWN_DEFAULT_PARAMS.items():
-        if field not in query_params:
-            query_params[field] = value
-
-
-def dedup_list(data, check_keys):
+class RestrictedViewMixin:
     """
-    Remove duplicate dictionaries from list.
+    This mixin overrides two helpers in order to pass the user object to the filters.
+    get_queryset alters the objects returned for list views.
+    get_form_kwargs passes the request object to the form class. Remaining permissions
+    checks are in forms.py
     """
-    seen = set()
-    out = []
 
-    dup_count = 0
-    for x in data:
-        dedupeKey = tuple(x[k] for k in check_keys if k in x)
-        if dedupeKey in seen:
-            dup_count += 1
-            continue
-        if dup_count > 0:
-            out.append({"type": "control", "hidden": dup_count})
-            dup_count = 0
-        out.append(x)
-        seen.add(dedupeKey)
-    if dup_count > 0:
-        out.append({"type": "control", "hidden": dup_count})
-    return out
+    allow_empty = True
+    queryset = None
+    model = None
+    paginate_by = None
+    paginate_orphans = 0
+    context_object_name = None
+    paginator_class = Paginator
+    page_kwarg = "page"
+    ordering = None
+
+    def get_queryset(self, **kwargs):
+        """
+        This function is overriden to filter the objects by the requesting user.
+        """
+        if self.queryset is not None:
+            queryset = self.queryset
+            if isinstance(queryset, QuerySet):
+                # queryset = queryset.all()
+                queryset = queryset.filter(user=self.request.user)
+        elif self.model is not None:
+            queryset = self.model._default_manager.filter(user=self.request.user)
+        else:
+            raise ImproperlyConfigured(
+                "%(cls)s is missing a QuerySet. Define "
+                "%(cls)s.model, %(cls)s.queryset, or override "
+                "%(cls)s.get_queryset()." % {"cls": self.__class__.__name__}
+            )
+        if hasattr(self, "get_ordering"):
+            ordering = self.get_ordering()
+            if ordering:
+                if isinstance(ordering, str):
+                    ordering = (ordering,)
+                queryset = queryset.order_by(*ordering)
+
+        return queryset
+
+    def get_form_kwargs(self):
+        """Passes the request object to the form class.
+        This is necessary to only display members that belong to a given user"""
+
+        kwargs = super().get_form_kwargs()
+        kwargs["request"] = self.request
+        return kwargs
+
+
+class ObjectNameMixin(object):
+    def __init__(self, *args, **kwargs):
+        if self.model is None:
+            self.title = self.context_object_name.title()
+            self.title_singular = self.context_object_name_singular.title()
+        else:
+            self.title_singular = self.model._meta.verbose_name.title()  # Hook
+            self.context_object_name_singular = self.title_singular.lower()  # hook
+            self.title = self.model._meta.verbose_name_plural.title()  # Hooks
+            self.context_object_name = self.title.lower()  # hooks
+
+            self.context_object_name = self.context_object_name.replace(" ", "")
+            self.context_object_name_singular = (
+                self.context_object_name_singular.replace(" ", "")
+            )
+        super().__init__(*args, **kwargs)
+
+
+class ObjectList(RestrictedViewMixin, ObjectNameMixin, ListView):
+    allowed_types = ["modal", "widget", "window", "page"]
+    window_content = "window-content/objects.html"
+    list_template = None
+
+    page_title = None
+    page_subtitle = None
+
+    list_url_name = None
+    # WARNING: TAKEN FROM locals()
+    list_url_args = ["type"]
+
+    submit_url_name = None
+
+    delete_all_url_name = None
+    widget_options = None
+
+    # copied from BaseListView
+    def get(self, request, *args, **kwargs):
+        type = kwargs.get("type", None)
+        if not type:
+            return HttpResponseBadRequest("No type specified")
+        if type not in self.allowed_types:
+            return HttpResponseBadRequest("Invalid type specified")
+
+        self.request = request
+        self.object_list = self.get_queryset(**kwargs)
+        if isinstance(self.object_list, HttpResponse):
+            return self.object_list
+        if isinstance(self.object_list, HttpResponseBadRequest):
+            return self.object_list
+        allow_empty = self.get_allow_empty()
+
+        self.template_name = f"wm/{type}.html"
+        unique = str(uuid.uuid4())[:8]
+
+        list_url_args = {}
+        for arg in self.list_url_args:
+            if arg in locals():
+                list_url_args[arg] = locals()[arg]
+            elif arg in kwargs:
+                list_url_args[arg] = kwargs[arg]
+
+        orig_type = type
+        if type == "page":
+            type = "modal"
+
+        if not allow_empty:
+            # When pagination is enabled and object_list is a queryset,
+            # it's better to do a cheap query than to load the unpaginated
+            # queryset in memory.
+            if self.get_paginate_by(self.object_list) is not None and hasattr(
+                self.object_list, "exists"
+            ):
+                is_empty = not self.object_list.exists()
+            else:
+                is_empty = not self.object_list
+            if is_empty:
+                raise Http404("Empty list")
+
+        context = self.get_context_data()
+        context["title"] = self.title + f" ({type})"
+        context["title_singular"] = self.title_singular
+        context["unique"] = unique
+        context["window_content"] = self.window_content
+        context["list_template"] = self.list_template
+        context["page_title"] = self.page_title
+        context["page_subtitle"] = self.page_subtitle
+        context["type"] = type
+        context["context_object_name"] = self.context_object_name
+        context["context_object_name_singular"] = self.context_object_name_singular
+
+        if self.submit_url_name is not None:
+            context["submit_url"] = reverse(self.submit_url_name, kwargs={"type": type})
+
+        if self.list_url_name is not None:
+            context["list_url"] = reverse(self.list_url_name, kwargs=list_url_args)
+
+        if self.delete_all_url_name:
+            context["delete_all_url"] = reverse(self.delete_all_url_name)
+        if self.widget_options:
+            context["widget_options"] = self.widget_options
+
+        # Return partials for HTMX
+        if self.request.htmx:
+            if request.headers["HX-Target"] == self.context_object_name + "-table":
+                self.template_name = self.list_template
+            elif orig_type == "page":
+                self.template_name = self.list_template
+            else:
+                context["window_content"] = self.list_template
+        return self.render_to_response(context)
+
+
+class ObjectCreate(RestrictedViewMixin, ObjectNameMixin, CreateView):
+    allowed_types = ["modal", "widget", "window", "page"]
+    window_content = "window-content/object-form.html"
+    parser_classes = [FormParser]
+
+    page_title = None
+    page_subtitle = None
+
+    model = None
+    submit_url_name = None
+    submit_url_args = ["type"]
+
+    request = None
+
+    # Whether to hide the cancel button in the form
+    hide_cancel = False
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.title = "Create " + self.context_object_name_singular
+
+    def post_save(self, obj):
+        pass
+
+    def form_valid(self, form):
+        obj = form.save(commit=False)
+        if self.request is None:
+            raise Exception("Request is None")
+        obj.user = self.request.user
+        obj.save()
+        form.save_m2m()
+        self.post_save(obj)
+        context = {"message": "Object created", "class": "success"}
+        response = self.render_to_response(context)
+        response["HX-Trigger"] = f"{self.context_object_name_singular}Event"
+        return response
+
+    def form_invalid(self, form):
+        """If the form is invalid, render the invalid form."""
+        return self.get(self.request, **self.kwargs, form=form)
+
+    def get(self, request, *args, **kwargs):
+        type = kwargs.get("type", None)
+        if not type:
+            return HttpResponseBadRequest("No type specified")
+        if type not in self.allowed_types:
+            return HttpResponseBadRequest("Invalid type specified")
+        self.template_name = f"wm/{type}.html"
+        unique = str(uuid.uuid4())[:8]
+
+        self.request = request
+        self.kwargs = kwargs
+
+        if type == "widget":
+            self.hide_cancel = True
+
+        if type == "page":
+            type = "modal"
+
+        self.object = None
+
+        submit_url_args = {}
+        for arg in self.submit_url_args:
+            if arg in locals():
+                submit_url_args[arg] = locals()[arg]
+            elif arg in kwargs:
+                submit_url_args[arg] = kwargs[arg]
+        submit_url = reverse(self.submit_url_name, kwargs=submit_url_args)
+
+        context = self.get_context_data()
+        form = kwargs.get("form", None)
+        if form:
+            context["form"] = form
+        context["unique"] = unique
+        context["window_content"] = self.window_content
+        context["context_object_name"] = self.context_object_name
+        context["context_object_name_singular"] = self.context_object_name_singular
+        context["submit_url"] = submit_url
+        context["type"] = type
+        context["hide_cancel"] = self.hide_cancel
+        if self.page_title:
+            context["page_title"] = self.page_title
+        if self.page_subtitle:
+            context["page_subtitle"] = self.page_subtitle
+        response = self.render_to_response(context)
+        # response["HX-Trigger"] = f"{self.context_object_name_singular}Event"
+        return response
+
+    def post(self, request, *args, **kwargs):
+        self.request = request
+        self.template_name = "partials/notify.html"
+        return super().post(request, *args, **kwargs)
+
+
+class ObjectRead(RestrictedViewMixin, ObjectNameMixin, DetailView):
+    allowed_types = ["modal", "widget", "window", "page"]
+    window_content = "window-content/object.html"
+    detail_template = "partials/generic-detail.html"
+
+    page_title = None
+    page_subtitle = None
+
+    model = None
+    # submit_url_name = None
+
+    detail_url_name = None
+    # WARNING: TAKEN FROM locals()
+    detail_url_args = ["type"]
+
+    request = None
+
+    def get(self, request, *args, **kwargs):
+        type = kwargs.get("type", None)
+        if not type:
+            return HttpResponseBadRequest("No type specified")
+        if type not in self.allowed_types:
+            return HttpResponseBadRequest()
+        self.template_name = f"wm/{type}.html"
+        unique = str(uuid.uuid4())[:8]
+
+        detail_url_args = {}
+        for arg in self.detail_url_args:
+            if arg in locals():
+                detail_url_args[arg] = locals()[arg]
+            elif arg in kwargs:
+                detail_url_args[arg] = kwargs[arg]
+
+        self.request = request
+        self.object = self.get_object(**kwargs)
+        if isinstance(self.object, HttpResponse):
+            return self.object
+
+        orig_type = type
+        if type == "page":
+            type = "modal"
+
+        context = self.get_context_data()
+
+        context["title"] = self.title + f" ({type})"
+        context["title_singular"] = self.title_singular
+        context["unique"] = unique
+        context["window_content"] = self.window_content
+        context["detail_template"] = self.detail_template
+        if self.page_title:
+            context["page_title"] = self.page_title
+        if self.page_subtitle:
+            context["page_subtitle"] = self.page_subtitle
+        context["type"] = type
+        context["context_object_name"] = self.context_object_name
+        context["context_object_name_singular"] = self.context_object_name_singular
+
+        if self.detail_url_name is not None:
+            context["detail_url"] = reverse(
+                self.detail_url_name, kwargs=detail_url_args
+            )
+
+        # Return partials for HTMX
+        if self.request.htmx:
+            if request.headers["HX-Target"] == self.context_object_name + "-info":
+                self.template_name = self.detail_template
+            elif orig_type == "page":
+                self.template_name = self.detail_template
+            else:
+                context["window_content"] = self.detail_template
+
+        return self.render_to_response(context)
+
+
+class ObjectUpdate(RestrictedViewMixin, ObjectNameMixin, UpdateView):
+    allowed_types = ["modal", "widget", "window", "page"]
+    window_content = "window-content/object-form.html"
+    parser_classes = [FormParser]
+
+    page_title = None
+    page_subtitle = None
+
+    model = None
+    submit_url_name = None
+    submit_url_args = ["type", "pk"]
+
+    request = None
+
+    # Whether pk is required in the get request
+    pk_required = True
+
+    # Whether to hide the cancel button in the form
+    hide_cancel = False
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.title = "Update " + self.context_object_name_singular
+
+    def post_save(self, obj):
+        pass
+
+    def form_valid(self, form):
+        obj = form.save(commit=False)
+        if self.request is None:
+            raise Exception("Request is None")
+        obj.save()
+        form.save_m2m()
+        self.post_save(obj)
+        context = {"message": "Object updated", "class": "success"}
+        response = self.render_to_response(context)
+        response["HX-Trigger"] = f"{self.context_object_name_singular}Event"
+        return response
+
+    def form_invalid(self, form):
+        """If the form is invalid, render the invalid form."""
+        return self.get(self.request, **self.kwargs, form=form)
+
+    def get(self, request, *args, **kwargs):
+        self.request = request
+        type = kwargs.get("type", None)
+        pk = kwargs.get("pk", None)
+        if not type:
+            return HttpResponseBadRequest("No type specified")
+        if not pk:
+            if self.pk_required:
+                return HttpResponseBadRequest("No pk specified")
+        if type not in self.allowed_types:
+            return HttpResponseBadRequest("Invalid type specified")
+        self.template_name = f"wm/{type}.html"
+        unique = str(uuid.uuid4())[:8]
+        if type == "widget":
+            self.hide_cancel = True
+
+        if type == "page":
+            type = "modal"
+
+        self.object = self.get_object()
+
+        submit_url_args = {}
+        for arg in self.submit_url_args:
+            if arg in locals():
+                submit_url_args[arg] = locals()[arg]
+            elif arg in kwargs:
+                submit_url_args[arg] = kwargs[arg]
+        submit_url = reverse(self.submit_url_name, kwargs=submit_url_args)
+
+        context = self.get_context_data()
+        form = kwargs.get("form", None)
+        if form:
+            context["form"] = form
+        context["title"] = self.title + f" ({type})"
+        context["title_singular"] = self.title_singular
+        context["unique"] = unique
+        context["window_content"] = self.window_content
+        context["context_object_name"] = self.context_object_name
+        context["context_object_name_singular"] = self.context_object_name_singular
+        context["submit_url"] = submit_url
+        context["type"] = type
+        context["hide_cancel"] = self.hide_cancel
+        if self.page_title:
+            context["page_title"] = self.page_title
+        if self.page_subtitle:
+            context["page_subtitle"] = self.page_subtitle
+        response = self.render_to_response(context)
+        # response["HX-Trigger"] = f"{self.context_object_name_singular}Event"
+        return response
+
+    def post(self, request, *args, **kwargs):
+        self.request = request
+        self.template_name = "partials/notify.html"
+        return super().post(request, *args, **kwargs)
+
+
+class ObjectDelete(RestrictedViewMixin, ObjectNameMixin, DeleteView):
+    model = None
+    template_name = "partials/notify.html"
+
+    # Overriden to prevent success URL from being used
+    def delete(self, request, *args, **kwargs):
+        """
+        Call the delete() method on the fetched object and then redirect to the
+        success URL.
+        """
+        self.object = self.get_object()
+        # success_url = self.get_success_url()
+        self.object.delete()
+        context = {"message": "Object deleted", "class": "success"}
+        response = self.render_to_response(context)
+        response["HX-Trigger"] = f"{self.context_object_name_singular}Event"
+        return response
+
+    # This will be used in newer Django versions, until then we get a warning
+    def form_valid(self, form):
+        """
+        Call the delete() method on the fetched object.
+        """
+        self.object = self.get_object()
+        self.object.delete()
+        context = {"message": "Object deleted", "class": "success"}
+        response = self.render_to_response(context)
+        response["HX-Trigger"] = f"{self.context_object_name_singular}Event"
+        return response
 
 
 # from random import randint

core/views/notifications.py

@@ -0,0 +1,59 @@
+from django.contrib.auth.mixins import LoginRequiredMixin
+
+from core.forms import NotificationRuleForm, NotificationSettingsForm
+from core.models import NotificationRule, NotificationSettings
+from core.views.helpers import ObjectCreate, ObjectDelete, ObjectList, ObjectUpdate
+
+
+# Notifications - we create a new notification settings object if there isn't one
+# Hence, there is only an update view, not a create view.
+class NotificationsUpdate(LoginRequiredMixin, ObjectUpdate):
+    model = NotificationSettings
+    form_class = NotificationSettingsForm
+
+    page_title = "Update your notification settings"
+    page_subtitle = (
+        "At least the topic must be set if you want to receive notifications."
+    )
+
+    submit_url_name = "notifications_update"
+    submit_url_args = ["type"]
+
+    pk_required = False
+
+    hide_cancel = True
+
+    def get_object(self, **kwargs):
+        notification_settings, _ = NotificationSettings.objects.get_or_create(
+            user=self.request.user
+        )
+        return notification_settings
+
+
+class RuleList(LoginRequiredMixin, ObjectList):
+    list_template = "partials/rule-list.html"
+    model = NotificationRule
+    page_title = "List of notification rules."
+
+    list_url_name = "rules"
+    list_url_args = ["type"]
+
+    submit_url_name = "rule_create"
+
+
+class RuleCreate(LoginRequiredMixin, ObjectCreate):
+    model = NotificationRule
+    form_class = NotificationRuleForm
+
+    submit_url_name = "rule_create"
+
+
+class RuleUpdate(LoginRequiredMixin, ObjectUpdate):
+    model = NotificationRule
+    form_class = NotificationRuleForm
+
+    submit_url_name = "rule_update"
+
+
+class RuleDelete(LoginRequiredMixin, ObjectDelete):
+    model = NotificationRule

core/views/ui/drilldown.py

@@ -6,11 +6,11 @@ from django.conf import settings
 from django.http import HttpResponse, JsonResponse
 from django.shortcuts import render
 from django.urls import reverse
-from django.views import View
 from django_tables2 import SingleTableView
 from rest_framework.parsers import FormParser
 from rest_framework.views import APIView
 
+from core.db import add_defaults, remove_defaults
 from core.db.storage import db
 from core.lib.threshold import (
     annotate_num_chans,
@@ -18,7 +18,6 @@ from core.lib.threshold import (
     get_chans,
     get_users,
 )
-from core.views import helpers
 from core.views.ui.tables import DrilldownTable
 
 # from copy import deepcopy
@@ -97,31 +96,46 @@ def make_graph(results):
     return orjson.dumps(graph).decode("utf-8")
 
 
-def drilldown_search(request, return_context=False, template=None):
+class DrilldownTableView(SingleTableView):
+    table_class = DrilldownTable
+    template_name = "wm/widget.html"
+    window_content = "window-content/results.html"
+    # htmx_partial = "partials/"
+    paginate_by = settings.DRILLDOWN_RESULTS_PER_PAGE
+
+    def common_request(self, request, **kwargs):
         extra_params = {}
 
-    if not template:
-        template_name = "widgets/table_results.html"
-    else:
-        template_name = template
         if request.user.is_anonymous:
             sizes = settings.MAIN_SIZES_ANON
         else:
             sizes = settings.MAIN_SIZES
 
         if request.GET:
-        if not request.htmx:
-            template_name = "ui/drilldown/drilldown.html"
+            self.template_name = "index.html"
+            # GET arguments in URL like ?query=xyz
             query_params = request.GET.dict()
+            if request.htmx:
+                if request.resolver_match.url_name == "search_partial":
+                    self.template_name = "partials/results_table.html"
         elif request.POST:
             query_params = request.POST.dict()
         else:
-        template_name = "ui/drilldown/drilldown.html"
+            self.template_name = "index.html"
+            # No query, this is a fresh page load
+            # Don't try to search, since there's clearly nothing to do
             params_with_defaults = {}
-        helpers.add_defaults(params_with_defaults)
-        context = {"sizes": sizes, "unique": "results", "params": params_with_defaults}
-        return render(request, template_name, context)
+            add_defaults(params_with_defaults)
+            context = {
+                "sizes": sizes,
+                "params": params_with_defaults,
+                "unique": "results",
+                "window_content": self.window_content,
+                "title": "Results",
+            }
+            return render(request, self.template_name, context)
 
+        # Merge everything together just in case
         tmp_post = request.POST.dict()
         tmp_get = request.GET.dict()
         tmp_post = {k: v for k, v in tmp_post.items() if v and not v == "None"}
@@ -139,47 +153,60 @@ def drilldown_search(request, return_context=False, template=None):
             del query_params["dates"]
             if dates:
                 if "message" in dates:
-                return render(request, template_name, dates)
+                    return render(request, self.template_name, dates)
                 query_params["from_date"] = dates["from_date"]
                 query_params["to_date"] = dates["to_date"]
                 query_params["from_time"] = dates["from_time"]
                 query_params["to_time"] = dates["to_time"]
 
-    if "query" in query_params:
         # Remove null values
+        if "query" in query_params:
             if query_params["query"] == "":
                 del query_params["query"]
-            # Turn the query into tags for populating the taglist
-            # tags = create_tags(query_params["query"])
-            # context["tags"] = tags
-    # else:
-    #     context = {"object_list": []}
 
+        # Remove null tags values
         if "tags" in query_params:
             if query_params["tags"] == "":
                 del query_params["tags"]
             else:
+                # Parse the tags and populate cast to pass to search function
                 tags = parse_tags(query_params["tags"])
                 extra_params["tags"] = tags
 
+        if "dedup" in query_params:
+            if query_params["dedup"] == "on":
+                extra_params["dedup"] = True
+            else:
+                extra_params["dedup"] = False
+        else:
+            extra_params["dedup"] = False
+
         context = db.query_results(request, query_params, **extra_params)
+
+        # Unique is for identifying the widgets.
+        # We don't want a random one since we only want one results pane.
         context["unique"] = "results"
+        context["window_content"] = self.window_content
+        context["title"] = "Results"
 
         # Valid sizes
         context["sizes"] = sizes
 
         # Add any default parameters to the context
         params_with_defaults = dict(query_params)
-    helpers.add_defaults(params_with_defaults)
+        add_defaults(params_with_defaults)
         context["params"] = params_with_defaults
 
-    helpers.remove_defaults(query_params)
+        # Remove anything that we or the user set to a default for
+        # pretty URLs
+        remove_defaults(query_params)
         url_params = urllib.parse.urlencode(query_params)
         context["client_uri"] = url_params
+
+        # There's an error
         if "message" in context:
-        if return_context:
-            return context
-        response = render(request, template_name, context)
+            response = render(request, self.template_name, context)
+            # Still push the URL so they can share it to get assistance
             if request.GET:
                 if request.htmx:
                     response["HX-Push"] = reverse("home") + "?" + url_params
@@ -191,6 +218,7 @@ def drilldown_search(request, return_context=False, template=None):
         graph = make_graph(context["object_list"])
         context["data"] = graph
 
+        # Create the table
         context = make_table(context)
 
         # URI we're passing to the template for linking, table fields removed
@@ -200,36 +228,14 @@ def drilldown_search(request, return_context=False, template=None):
         context["uri"] = clean_url_params
 
         # unique = str(uuid.uuid4())[:8]
-    if return_context:
+        # self.context = context
         return context
 
-    response = render(request, template_name, context)
-    if request.GET:
-        if request.htmx:
-            response["HX-Push"] = reverse("home") + "?" + url_params
-    elif request.POST:
-        response["HX-Push"] = reverse("home") + "?" + url_params
-
-    return response
-
-
-class DrilldownTableView(SingleTableView):
-    table_class = DrilldownTable
-    template_name = "widgets/table_results.html"
-    paginate_by = settings.DRILLDOWN_RESULTS_PER_PAGE
-
-    def get_queryset(self, request, **kwargs):
-        context = drilldown_search(request, return_context=True)
-        # Save the context as we will need to merge other attributes later
-        self.context = context
-
-        if "object_list" in context:
-            return context["object_list"]
-        else:
-            return []
-
     def get(self, request, *args, **kwargs):
-        self.object_list = self.get_queryset(request)
+        self.context = self.common_request(request)
+        if isinstance(self.context, HttpResponse):
+            return self.context
+        self.object_list = self.context["object_list"]
         show = []
         show = set().union(*(d.keys() for d in self.object_list))
         allow_empty = self.get_allow_empty()
@@ -245,17 +251,17 @@ class DrilldownTableView(SingleTableView):
             else:
                 is_empty = not self.object_list  # noqa
         context = self.get_context_data()
-        if isinstance(self.context, HttpResponse):
-            return self.context
 
         for k, v in self.context.items():
             if k not in context:
                 context[k] = v
         context["show"] = show
 
-        if request.method == "GET":
-            if not request.htmx:
-                self.template_name = "ui/drilldown/drilldown.html"
+        # if request.htmx:
+        #    self.template_name = self.window_content
+        # if request.method == "GET":
+        #     if not request.htmx:
+        #         self.template_name = "ui/drilldown/drilldown.html"
         response = self.render_to_response(context)
         # if not request.method == "GET":
         if "client_uri" in context:
@@ -266,15 +272,15 @@ class DrilldownTableView(SingleTableView):
         return self.get(request, *args, **kwargs)
 
 
-class Drilldown(View):
-    template_name = "ui/drilldown/drilldown.html"
-    plan_name = "drilldown"
+# class Drilldown(View):
+#     template_name = "ui/drilldown/drilldown.html"
+#     plan_name = "drilldown"
 
-    def get(self, request):
-        return drilldown_search(request)
+#     def get(self, request):
+#         return drilldown_search(request)
 
-    def post(self, request):
-        return drilldown_search(request)
+#     def post(self, request):
+#         return drilldown_search(request)
 
 
 class DrilldownContextModal(APIView):
@@ -389,19 +395,6 @@ class DrilldownContextModal(APIView):
         if "message" in results:
             return render(request, self.template_name, results)
 
-        # if settings.HASHING:  # we probably want to see the tokens
-        #     if query_params["source"] not in settings.SAFE_SOURCES:
-        #         if not request.user.has_perm("core.bypass_hashing"):
-        #             for index, item in enumerate(results["object_list"]):
-        #                 if "tokens" in item:
-        #                     results["object_list"][index]["msg"] = results[
-        #                         "object_list"
-        #                     ][index].pop("tokens")
-        #                     # item["msg"] = item.pop("tokens")
-
-        # Make the time nicer
-        # for index, item in enumerate(results["object_list"]):
-        #    results["object_list"][index]["time"] = item["time"]+"SSS"
         unique = str(uuid.uuid4())[:8]
         context = {
             "net": query_params["net"],
@@ -449,45 +442,18 @@ class ThresholdInfoModal(APIView):
         nick = request.data["nick"]
         channel = request.data["channel"]
 
-        # SAFE BLOCK #
-        # Lookup the hash values but don't disclose them to the user
-        # if settings.HASHING:
-        #     SAFE_PARAMS = request.data.dict()
-        #     hash_lookup(request.user, SAFE_PARAMS)
-
         channels = get_chans(net, [nick])
-        print("CHANNELS", channels)
         users = get_users(net, [nick])
-        print("USERS", users)
         num_users = annotate_num_users(net, channels)
-        print("NUM_USERS", num_users)
         num_chans = annotate_num_chans(net, users)
-        print("NUM_CHANS", num_chans)
         if channels:
             inter_users = get_users(net, channels)
         else:
             inter_users = []
-        print("INTER_USERS", inter_users)
         if users:
             inter_chans = get_chans(net, users)
         else:
             inter_chans = []
-        print("INTER_CHANS", inter_chans)
-        # if settings.HASHING:
-        #     hash_list(request.user, inter_chans)
-        #     hash_list(request.user, inter_users)
-
-        #     hash_list(request.user, num_chans, hash_keys=True)
-        #     hash_list(request.user, num_users, hash_keys=True)
-
-        #     hash_list(request.user, channels)
-        #     hash_list(request.user, users)
-
-        # if settings.RANDOMISATION:
-        #     randomise_list(request.user, num_chans)
-        #     randomise_list(request.user, num_users)
-
-        # SAFE BLOCK END #
 
         unique = str(uuid.uuid4())[:8]
         context = {
@@ -502,5 +468,4 @@ class ThresholdInfoModal(APIView):
             "num_chans": num_chans,
             "unique": unique,
         }
-        print("CON", context)
         return render(request, self.template_name, context)

core/views/ui/tables.py

@@ -64,14 +64,13 @@ class DrilldownTable(Table):
     mtype = Column()
     realname = Column()
     server = Column()
-    mtype = Column()
-    # tokens = Column()
+    tokens = Column()
     lang_code = Column()
     lang_name = Column()
-    words_noun = Column()
-    words_adj = Column()
-    words_verb = Column()
-    words_adv = Column()
+    # words_noun = Column()
+    # words_adj = Column()
+    # words_verb = Column()
+    # words_adv = Column()
     hidden = Column()
     filename = Column()
     file_md5 = Column()

docker-compose.yml

@@ -1,17 +1,21 @@
-version: "2"
+version: "2.2"
 
 services:
   app:
     image: pathogen/neptune:latest
-    build: ./docker
+    container_name: neptune
+    build:
+      context: .
+      args:
+        OPERATION: ${OPERATION}
     volumes:
       - ${PORTAINER_GIT_DIR}:/code
-      - ${NEPTUNE_LOCAL_SETTINGS}:/code/app/local_settings.py
-      - ${NEPTUNE_DATABASE_FILE}:/code/db.sqlite3
-    ports:
-      - "${NEPTUNE_PORT}:8000"
+      - ${PORTAINER_GIT_DIR}/docker/uwsgi.ini:/conf/uwsgi.ini
+      - ${APP_LOCAL_SETTINGS}:/code/app/local_settings.py
+      - ${APP_DATABASE_FILE}:/code/db.sqlite3
+      - neptune_static:${STATIC_ROOT}
     env_file:
-      - .env
+      - stack.env
     volumes_from:
       - tmp
     depends_on:
@@ -19,49 +23,140 @@ services:
         condition: service_healthy
       migration:
         condition: service_started
+      collectstatic:
+        condition: service_started
+    networks:
+      - default
+      - pathogen
+      - elastic
+
+  processing:
+    image: pathogen/neptune:latest
+    container_name: processing_neptune
+    build:
+      context: .
+      args:
+        OPERATION: ${OPERATION}
+    command: sh -c '. /venv/bin/activate && python manage.py processing'
+    volumes:
+      - ${PORTAINER_GIT_DIR}:/code
+      - ${PORTAINER_GIT_DIR}/docker/uwsgi.ini:/conf/uwsgi.ini
+      - ${APP_LOCAL_SETTINGS}:/code/app/local_settings.py
+      - ${APP_DATABASE_FILE}:/code/db.sqlite3
+      - neptune_static:${STATIC_ROOT}
+    env_file:
+      - stack.env
+    volumes_from:
+      - tmp
+    depends_on:
+      redis:
+        condition: service_healthy
+      migration:
+        condition: service_started
+      collectstatic:
+        condition: service_started
+    networks:
+      - default
 
   migration:
     image: pathogen/neptune:latest
+    container_name: migration_neptune
+    build:
+      context: .
+      args:
+        OPERATION: ${OPERATION}
     command: sh -c '. /venv/bin/activate && python manage.py migrate --noinput'
     volumes:
       - ${PORTAINER_GIT_DIR}:/code
-      - ${NEPTUNE_LOCAL_SETTINGS}:/code/app/local_settings.py
-      - ${NEPTUNE_DATABASE_FILE}:/code/db.sqlite3
+      - ${APP_LOCAL_SETTINGS}:/code/app/local_settings.py
+      - ${APP_DATABASE_FILE}:/code/db.sqlite3
+      - neptune_static:${STATIC_ROOT}
     volumes_from:
       - tmp
     depends_on:
       redis:
         condition: service_healthy
 
-  # pyroscope:
-  #   image: pyroscope/pyroscope
-  #   environment:
-  #     - PYROSCOPE_LOG_LEVEL=debug
-  #   ports:
-  #     - '4040:4040'
-  #   command:
-  #     - 'server'
+  collectstatic:
+    image: pathogen/neptune:latest
+    container_name: collectstatic_neptune
+    build:
+      context: .
+      args:
+        OPERATION: ${OPERATION}
+    command: sh -c '. /venv/bin/activate && python manage.py collectstatic --noinput'
+    volumes:
+      - ${PORTAINER_GIT_DIR}:/code
+      - ${APP_LOCAL_SETTINGS}:/code/app/local_settings.py
+      - ${APP_DATABASE_FILE}:/code/db.sqlite3
+      - neptune_static:${STATIC_ROOT}
+    volumes_from:
+      - tmp
+    env_file:
+      - stack.env
+    depends_on:
+      redis:
+        condition: service_healthy
+
+  nginx:
+    image: nginx:latest
+    container_name: nginx_neptune
+    ports:
+      - ${APP_PORT}:9999
+    ulimits:
+      nproc: 65535
+      nofile:
+         soft: 65535
+         hard: 65535
+    volumes:
+      - ${PORTAINER_GIT_DIR}:/code
+      - ${PORTAINER_GIT_DIR}/docker/nginx/conf.d/${OPERATION}.conf:/etc/nginx/conf.d/default.conf
+      - neptune_static:${STATIC_ROOT}
+    volumes_from:
+      - tmp
+    networks:
+      - default
+      - pathogen
+    depends_on:
+      app:
+        condition: service_started
 
   tmp:
     image: busybox
-    command: chmod -R 777 /var/run/redis
+    container_name: tmp_neptune
+    command: chmod -R 777 /var/run/socks
     volumes:
-      - /var/run/redis
+      - /var/run/socks
 
   redis:
     image: redis
+    container_name: redis_neptune
     command: redis-server /etc/redis.conf
+    ulimits:
+      nproc: 65535
+      nofile:
+         soft: 65535
+         hard: 65535
     volumes:
       - ${PORTAINER_GIT_DIR}/docker/redis.conf:/etc/redis.conf
     volumes_from:
       - tmp
     healthcheck:
-      test: "redis-cli -s /var/run/redis/redis.sock ping"
+      test: "redis-cli -s /var/run/socks/redis.sock ping"
       interval: 2s
       timeout: 2s
       retries: 15
+    networks:
+      - default
+      - pathogen
 
 networks:
   default:
-      external:
-         name: pathogen
+    driver: bridge
+  pathogen:
+    external: true
+  elastic:
+    external: true
+
+volumes:
+  neptune_static: {}

docker/docker-compose.prod.yml

@@ -1,60 +0,0 @@
-version: "2"
-   
-services:
-  app:
-    image: pathogen/neptune:latest
-    build: ./docker/prod
-    volumes:
-      - ${PORTAINER_GIT_DIR}:/code
-      - ${PORTAINER_GIT_DIR}/docker/prod/uwsgi.ini:/conf/uwsgi.ini
-      - ${NEPTUNE_LOCAL_SETTINGS}:/code/app/local_settings.py
-      - ${NEPTUNE_DATABASE_FILE}:/code/db.sqlite3
-    ports:
-      - "${NEPTUNE_PORT}:8000" # uwsgi socket
-    env_file:
-      - ../stack.env
-    volumes_from:
-      - tmp
-    depends_on:
-      redis:
-        condition: service_healthy
-      migration:
-        condition: service_started
-
-  migration:
-    image: pathogen/neptune:latest
-    build: ./docker/prod
-    command: sh -c '. /venv/bin/activate && python manage.py migrate --noinput'
-    volumes:
-      - ${PORTAINER_GIT_DIR}:/code
-      - ${NEPTUNE_LOCAL_SETTINGS}:/code/app/local_settings.py
-      - ${NEPTUNE_DATABASE_FILE}:/code/db.sqlite3
-    volumes_from:
-      - tmp
-    depends_on:
-      redis:
-        condition: service_healthy
-
-  tmp:
-    image: busybox
-    command: chmod -R 777 /var/run/redis
-    volumes:
-      - /var/run/redis
-
-  redis:
-    image: redis
-    command: redis-server /etc/redis.conf
-    volumes:
-        - ${PORTAINER_GIT_DIR}/docker/redis.conf:/etc/redis.conf
-    volumes_from:
-        - tmp
-    healthcheck:
-      test: "redis-cli -s /var/run/redis/redis.sock ping"
-      interval: 2s
-      timeout: 2s
-      retries: 15
-
-networks:
-   default:
-      external:
-         name: pathogen

docker/nginx/conf.d/dev.conf

@@ -0,0 +1,23 @@
+upstream django {
+    #server app:8000;
+    #server unix:///var/run/socks/app.sock;
+    server app:8000;
+}
+
+server {
+  listen 9999;
+
+  location = /favicon.ico { access_log off; log_not_found off; }
+
+  location /static/ {
+    root /conf;
+  }
+
+  location / {
+    proxy_pass  http://django;
+    proxy_set_header   X-Real-IP $remote_addr;
+    proxy_set_header   X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header   Host $host;
+  }
+
+}

docker/nginx/conf.d/uwsgi.conf

@@ -0,0 +1,24 @@
+upstream django {
+    server app:8000;
+    #server unix:///var/run/socks/app.sock;
+}
+
+server {
+  listen 9999;
+
+  location = /favicon.ico { access_log off; log_not_found off; }
+
+  location /static/ {
+    root /conf;
+  }
+
+  location / {
+      include     /etc/nginx/uwsgi_params; # the uwsgi_params file you installed
+      uwsgi_pass  django;
+      uwsgi_param Host $host;
+      uwsgi_param X-Real-IP $remote_addr;
+      uwsgi_param X-Forwarded-For $proxy_add_x_forwarded_for;
+      uwsgi_param X-Forwarded-Proto $http_x_forwarded_proto;
+  }
+
+}

docker/prod/Dockerfile

@@ -1,21 +0,0 @@
-# syntax=docker/dockerfile:1
-FROM python:3
-
-RUN useradd -d /code pathogen
-RUN mkdir /code
-RUN chown pathogen:pathogen /code
-
-RUN mkdir /conf
-RUN chown pathogen:pathogen /conf
-
-RUN mkdir /venv
-RUN chown pathogen:pathogen /venv
-
-USER pathogen
-ENV PYTHONDONTWRITEBYTECODE=1
-ENV PYTHONUNBUFFERED=1
-WORKDIR /code
-COPY requirements.prod.txt /code/
-RUN python -m venv /venv
-RUN . /venv/bin/activate && pip install -r requirements.prod.txt
-CMD . /venv/bin/activate && uwsgi --ini /conf/uwsgi.ini

docker/prod/requirements.prod.txt

@@ -1,19 +0,0 @@
-wheel
-django
-django-crispy-forms
-crispy-bulma
-#opensearch-py
-stripe
-django-rest-framework
-numpy
-uwsgi
-django-tables2
-django-tables2-bulma-template
-django-htmx
-cryptography
-siphashc
-redis
-sortedcontainers
-django-debug-toolbar
-django-debug-toolbar-template-profiler
-orjson

docker/redis.conf

@@ -1,2 +1,5 @@
-unixsocket /var/run/redis/redis.sock
+unixsocket /var/run/socks/redis.sock
 unixsocketperm 777
+
+# For Monolith PubSub
+port 6379

docker/requirements.dev.txt

@@ -1,18 +0,0 @@
-wheel
-django
-django-crispy-forms
-crispy-bulma
-#opensearch-py
-stripe
-django-rest-framework
-numpy
-django-tables2
-django-tables2-bulma-template
-django-htmx
-cryptography
-siphashc
-redis
-sortedcontainers
-django-debug-toolbar
-django-debug-toolbar-template-profiler
-orjson

docker/uwsgi.ini

@@ -5,9 +5,8 @@ env=DJANGO_SETTINGS_MODULE=app.settings
 master=1
 pidfile=/tmp/project-master.pid
 socket=0.0.0.0:8000
-processes=5
 harakiri=20
-max-requests=5000
+max-requests=100000
 vacuum=1
 home=/venv
-
+processes=12

requirements.txt

@@ -1,9 +1,10 @@
 wheel
+uwsgi
 django
 pre-commit
 django-crispy-forms
 crispy-bulma
-#opensearch-py
+elasticsearch
 stripe
 django-rest-framework
 numpy
@@ -17,3 +18,4 @@ sortedcontainers
 django-debug-toolbar
 django-debug-toolbar-template-profiler
 orjson
+msgpack

stack.env

@@ -1,4 +1,6 @@
-NEPTUNE_PORT=5000
-PORTAINER_GIT_DIR=..
-NEPTUNE_LOCAL_SETTINGS=../app/local_settings.py
-NEPTUNE_DATABASE_FILE=../db.sqlite3
+APP_PORT=5000
+PORTAINER_GIT_DIR=.
+APP_LOCAL_SETTINGS=./app/local_settings.py
+APP_DATABASE_FILE=./db.sqlite3
+STATIC_ROOT=/conf/static
+OPERATION=dev