Remove unnecessary import in Threshold handler

.pre-commit-config.yaml

@@ -15,8 +15,17 @@ repos:
       - id: flake8
         args: [--max-line-length=88]
         exclude: ^core/migrations
-  - repo: https://github.com/thibaudcolas/curlylint
-    rev: v0.13.1
+  - repo: https://github.com/rtts/djhtml
+    rev: 'v1.5.2'  # replace with the latest tag on GitHub
     hooks:
-      - id: curlylint
-        files: \.(html|sls)$
+      - id: djhtml
+        args: [-t 2]
+      - id: djcss
+        exclude : ^core/static/css # slow
+      - id: djjs
+        exclude: ^core/static/js # slow
+#  - repo: https://github.com/thibaudcolas/curlylint
+#    rev: v0.13.1
+#    hooks:
+#      - id: curlylint
+#        files: \.(html|sls)$

app/local_settings.example.py

@@ -1,29 +1,70 @@
-# Secret key
-SECRET_KEY = ""
-
-# SECURITY WARNING: don't run with debug turned on in production!
-DEBUG = True
-
 # OpenSearch settings
 OPENSEARCH_URL = "127.0.0.1"
 OPENSEARCH_PORT = 9200
 OPENSEARCH_TLS = True
-OPENSEARCH_USERNAME = "opensearch_user1"
-OPENSEARCH_PASSWORD = "hunter2"
+OPENSEARCH_USERNAME = "admin"
+OPENSEARCH_PASSWORD = ""
 
-OPENSEARCH_INDEX_MAIN = "main"
-OPENSEARCH_INDEX_META = "meta"
+OPENSEARCH_INDEX_MAIN = "pathogen-main"
+OPENSEARCH_INDEX_META = "pathogen-meta"
+OPENSEARCH_INDEX_INT = "pathogen-int"
 
-OPENSEARCH_MAIN_SEARCH_FIELDS = ["msg", "nick", "host", "ident"]
-OPENSEARCH_MAIN_SIZES = ["5", "10", "15", "20", "50", "100", "200"]
-OPENSEARCH_MAIN_TIMESCALES = ["minute", "hour", "day", "week", "month", "6months"]
+OPENSEARCH_MAIN_SIZES = ["20", "50", "100", "200", "400", "800"]
+OPENSEARCH_MAIN_SIZES_ANON = ["20", "50", "100"]
+OPENSEARCH_MAIN_SOURCES = ["dis", "4ch", "all"]
+OPENSEARCH_SOURCES_RESTRICTED = ["irc"]
 
-OPENSEARCH_BLACKLISTED = {
-    "msg": ["example.com"],
-    "nick": ["me"],
+# Manticore settings
+MANTICORE_URL = "http://monolith-db-1:9308"
+MANTICORE_INDEX_MAIN = "main"
+MANTICORE_INDEX_META = "meta"
+MANTICORE_INDEX_INT = "internal"
+
+MANTICORE_MAIN_SIZES = ["20", "50", "100", "200", "400", "800"]
+MANTICORE_MAIN_SIZES_ANON = ["20", "50", "100"]
+MANTICORE_MAIN_SOURCES = ["dis", "4ch", "all"]
+MANTICORE_SOURCES_RESTRICTED = ["irc"]
+MANTICORE_CACHE = True
+MANTICORE_CACHE_TIMEOUT = 60
+
+DRILLDOWN_RESULTS_PER_PAGE = 15
+DRILLDOWN_DEFAULT_PARAMS = {
+    "size": "20",
+    "index": "main",
+    "sorting": "desc",
+    "source": "4ch",
 }
 
-# URLs
+# Encryption
+# ENCRYPTION = False
+# ENCRYPTION_KEY = b""
+
+# Hashing
+# HASHING = True
+# HASHING_KEY = "xxx"
+
+# Obfuscation
+# OBFUSCATION = True
+# # Fields obfuscate based on separators
+# OBFUSCATE_FIELDS_SEP = ["date", "time"]
+# # Fields to obfuscate based on length
+# OBFUSCATE_FIELDS = ["ts"]
+# OBFUSCATE_KEEP_RATIO = 0.9
+# # DON'T obfuscate the last X fields of values separates by dashes
+# OBFUSCATE_DASH_NUM = 2
+# # DON'T obfuscate the last X fields of values separates by colons
+# OBFUSCATE_COLON_NUM = 1
+
+# SEARCH_FIELDS_DENY = ["ts", "date", "time"]
+
+# DELAY_RESULTS = True
+# # Delay results by this many days
+# DELAY_DURATION = 10
+
+OPENSEARCH_BLACKLISTED = {}
+
+
+# URLs\
 DOMAIN = "example.com"
 URL = f"https://{DOMAIN}"
 
@@ -35,23 +76,23 @@ CSRF_TRUSTED_ORIGINS = [URL]
 
 # Stripe
 STRIPE_TEST = True
-STRIPE_API_KEY_TEST = "sk_test_xxx"
-STRIPE_PUBLIC_API_KEY_TEST = "pk_test_xxx"
+STRIPE_API_KEY_TEST = ""
+STRIPE_PUBLIC_API_KEY_TEST = ""
 
-STRIPE_API_KEY_PROD = "sk_prod_xxx"
-STRIPE_PUBLIC_API_KEY_PROD = "pk_prod_xxx"
+STRIPE_API_KEY_PROD = ""
+STRIPE_PUBLIC_API_KEY_PROD = ""
 
 STRIPE_ENDPOINT_SECRET = ""
 STATIC_ROOT = ""
 SECRET_KEY = "a"
 
-STRIPE_ADMIN_COUPON = "promo"
+STRIPE_ADMIN_COUPON = ""
 
 # Threshold
-THRESHOLD_ENDPOINT = "http://127.0.0.1:13869"
-THRESHOLD_API_KEY = "name"
-THRESHOLD_API_TOKEN = "token"
-THRESHOLD_API_COUNTER = "counter"
+THRESHOLD_ENDPOINT = "http://threshold-app-1:13869"
+THRESHOLD_API_KEY = ""
+THRESHOLD_API_TOKEN = ""
+THRESHOLD_API_COUNTER = ""
 
 # NickTrace
 NICKTRACE_MAX_ITERATIONS = 4
@@ -64,4 +105,13 @@ META_MAX_CHUNK_SIZE = 500
 META_QUERY_SIZE = 10000
 
 DEBUG = True
-PROFILER = True
+PROFILER = False
+
+if DEBUG:
+    import socket  # only if you haven't already imported this
+
+    hostname, _, ips = socket.gethostbyname_ex(socket.gethostname())
+    INTERNAL_IPS = [ip[: ip.rfind(".")] + ".1" for ip in ips] + [
+        "127.0.0.1",
+        "10.0.2.2",
+    ]

app/settings.py

@@ -35,6 +35,8 @@ INSTALLED_APPS = [
     "django.contrib.sessions",
     "django.contrib.messages",
     "django.contrib.staticfiles",
+    "debug_toolbar",
+    "template_profiler_panel",
     "django_htmx",
     "crispy_forms",
     "crispy_bulma",
@@ -46,6 +48,7 @@ CRISPY_ALLOWED_TEMPLATE_PACKS = ("bulma",)
 DJANGO_TABLES2_TEMPLATE = "django-tables2/bulma.html"
 
 MIDDLEWARE = [
+    "debug_toolbar.middleware.DebugToolbarMiddleware",
     "django.middleware.security.SecurityMiddleware",
     "django.contrib.sessions.middleware.SessionMiddleware",
     "django.middleware.common.CommonMiddleware",
@@ -139,6 +142,29 @@ REST_FRAMEWORK = {
     ]
 }
 
+INTERNAL_IPS = [
+    "127.0.0.1",
+    "10.1.10.11",
+]
+
+DEBUG_TOOLBAR_PANELS = [
+    "template_profiler_panel.panels.template.TemplateProfilerPanel",
+    "debug_toolbar.panels.history.HistoryPanel",
+    "debug_toolbar.panels.versions.VersionsPanel",
+    "debug_toolbar.panels.timer.TimerPanel",
+    "debug_toolbar.panels.settings.SettingsPanel",
+    "debug_toolbar.panels.headers.HeadersPanel",
+    "debug_toolbar.panels.request.RequestPanel",
+    "debug_toolbar.panels.sql.SQLPanel",
+    "debug_toolbar.panels.staticfiles.StaticFilesPanel",
+    "debug_toolbar.panels.templates.TemplatesPanel",
+    "debug_toolbar.panels.cache.CachePanel",
+    "debug_toolbar.panels.signals.SignalsPanel",
+    "debug_toolbar.panels.logging.LoggingPanel",
+    "debug_toolbar.panels.redirects.RedirectsPanel",
+    "debug_toolbar.panels.profiling.ProfilingPanel",
+]
+
 from app.local_settings import *  # noqa
 
 if PROFILER:  # noqa - trust me its there

app/urls.py

@@ -20,27 +20,34 @@ from django.urls import include, path
 from django.views.generic import TemplateView
 
 # Threshold API stuff
-from core.api.views.threshold import ThresholdChans, ThresholdOnline, ThresholdUsers
 from core.views import About, Billing, Cancel, Order, Portal, Signup
 from core.views.callbacks import Callback
+from core.views.manage.threshold.irc import (
+    ThresholdIRCNetworkList,  # Actions and just get list output
+)
 from core.views.manage.threshold.irc import (
     ThresholdIRCActions,
     ThresholdIRCActionsAddNetwork,
+    ThresholdIRCActionsRegistration,
+    ThresholdIRCActionsRegistrationAuth,
     ThresholdIRCAliases,
     ThresholdIRCAliasesEdit,
     ThresholdIRCNetworkActions,
     ThresholdIRCNetworkActionsAuto,
-    ThresholdIRCNetworkActionsList,
     ThresholdIRCNetworkActionsRelay,
     ThresholdIRCNetworkChannels,
+    ThresholdIRCNetworkChannelsAPI,
     ThresholdIRCNetworkDel,
     ThresholdIRCNetworkInfo,
     ThresholdIRCNetworkInfoEdit,
+    ThresholdIRCNetworkRelayAuth,
     ThresholdIRCNetworkRelayDel,
+    ThresholdIRCNetworkRelayProvision,
     ThresholdIRCNetworkRelays,
     ThresholdIRCNetworkRelayStatus,
     ThresholdIRCNetworks,
     ThresholdIRCOverviewAlerts,
+    ThresholdIRCSendMessage,
     ThresholdIRCStats,
 )
 
@@ -51,18 +58,25 @@ from core.views.manage.threshold.threshold import (
 )
 
 # Main tool pages
-from core.views.ui.drilldown import Drilldown, ThresholdInfoModal  # DrilldownTableView,
-from core.views.ui.insights import (
-    Insights,
-    InsightsChannels,
-    InsightsInfoModal,
-    InsightsMeta,
-    InsightsNicks,
-    InsightsSearch,
+from core.views.ui.drilldown import (  # DrilldownTableView,; Drilldown,
+    DrilldownContextModal,
+    DrilldownTableView,
+    ThresholdInfoModal,
 )
 
+# from core.views.ui.insights import (
+#     Insights,
+#     InsightsChannels,
+#     InsightsInfoModal,
+#     InsightsMeta,
+#     InsightsNicks,
+#     InsightsSearch,
+# )
+
 urlpatterns = [
-    path("", Drilldown.as_view(), name="home"),
+    path("__debug__/", include("debug_toolbar.urls")),
+    path("", DrilldownTableView.as_view(), name="home"),
+    path("search/", DrilldownTableView.as_view(), name="search"),
     path("about/", About.as_view(), name="about"),
     path("callback", Callback.as_view(), name="callback"),
     path("billing/", Billing.as_view(), name="billing"),
@@ -83,13 +97,16 @@ urlpatterns = [
     ##
     # path("drilldown/", Drilldown.as_view(), name="drilldown"),
     path("modal/", ThresholdInfoModal.as_view(), name="modal_drilldown"),
+    path("modal/<str:type>/", ThresholdInfoModal.as_view(), name="modal_drilldown"),
+    path("context/", DrilldownContextModal.as_view(), name="modal_context"),
+    path("context_table/", DrilldownContextModal.as_view(), name="modal_context_table"),
     ##
-    path("ui/insights/", Insights.as_view(), name="insights"),
-    path("ui/insights/search/", InsightsSearch.as_view(), name="search_insights"),
-    path("ui/insights/channels/", InsightsChannels.as_view(), name="chans_insights"),
-    path("ui/insights/nicks/", InsightsNicks.as_view(), name="nicks_insights"),
-    path("ui/insights/meta/", InsightsMeta.as_view(), name="meta_insights"),
-    path("ui/insights/modal/", InsightsInfoModal.as_view(), name="modal_insights"),
+    # path("ui/insights/", Insights.as_view(), name="insights"),
+    # path("ui/insights/search/", InsightsSearch.as_view(), name="search_insights"),
+    # path("ui/insights/channels/", InsightsChannels.as_view(), name="chans_insights"),
+    # path("ui/insights/nicks/", InsightsNicks.as_view(), name="nicks_insights"),
+    # path("ui/insights/meta/", InsightsMeta.as_view(), name="meta_insights"),
+    # path("ui/insights/modal/", InsightsInfoModal.as_view(), name="modal_insights"),
     ##
     path(
         "manage/threshold/irc/overview/",
@@ -121,6 +138,31 @@ urlpatterns = [
         ThresholdIRCActionsAddNetwork.as_view(),
         name="threshold_irc_actions_add-network",
     ),
+    path(
+        "manage/threshold/irc/actions/registration/auth/",
+        ThresholdIRCActionsRegistrationAuth.as_view(),
+        name="threshold_irc_actions_registration_auth",
+    ),
+    path(
+        "manage/threshold/irc/actions/registration/<str:net>/",
+        ThresholdIRCActionsRegistration.as_view(),
+        name="threshold_irc_actions_registration_net",
+    ),
+    path(
+        "manage/threshold/irc/network/<str:net>/<int:num>/provision/",
+        ThresholdIRCNetworkRelayProvision.as_view(),
+        name="threshold_irc_network_relay_provision",
+    ),
+    path(
+        "manage/threshold/irc/network/<str:net>/<int:num>/auth/",
+        ThresholdIRCNetworkRelayAuth.as_view(),
+        name="threshold_irc_network_relay_auth",
+    ),
+    path(
+        "manage/threshold/irc/actions/registration/",
+        ThresholdIRCActionsRegistration.as_view(),
+        name="threshold_irc_actions_registration",
+    ),
     path(
         "manage/threshold/irc/network/<str:net>/",
         ThresholdIRCNetwork.as_view(),
@@ -166,6 +208,11 @@ urlpatterns = [
         ThresholdIRCNetworkChannels.as_view(),
         name="threshold_irc_network_channels",
     ),
+    path(
+        "manage/threshold/irc/network/<str:net>/channel/json/",
+        ThresholdIRCNetworkChannelsAPI.as_view(),
+        name="threshold_irc_network_channel_json",
+    ),
     path(
         "manage/threshold/irc/network/<str:net>/channel/<channel>/",
         ThresholdIRCNetworkChannels.as_view(),
@@ -204,11 +251,13 @@ urlpatterns = [
     ),
     path(
         "manage/threshold/irc/list/<str:net>/",
-        ThresholdIRCNetworkActionsList.as_view(),
-        name="threshold_irc_network_actions_list",
+        ThresholdIRCNetworkList.as_view(),
+        name="threshold_irc_network_list",
+    ),
+    path(
+        "manage/threshold/irc/msg/<str:net>/<str:num>/",
+        ThresholdIRCSendMessage.as_view(),
+        name="threshold_irc_msg",
     ),
     ##
-    path("api/chans/", ThresholdChans.as_view(), name="chans"),
-    path("api/users/", ThresholdUsers.as_view(), name="users"),
-    path("api/online/", ThresholdOnline.as_view(), name="online"),
 ] + static(settings.STATIC_URL, document_root=settings.STATIC_ROOT)

core/__init__.py

@@ -1,5 +1,8 @@
 import stripe
 from django.conf import settings
+from redis import StrictRedis
+
+r = StrictRedis(unix_socket_path="/var/run/redis/redis.sock", db=0)
 
 if settings.STRIPE_TEST:
     stripe.api_key = settings.STRIPE_API_KEY_TEST

core/api/views/threshold.py

@@ -1,67 +0,0 @@
-import logging
-
-from django.contrib.auth.mixins import LoginRequiredMixin
-from django.http import HttpResponse, JsonResponse
-from rest_framework.parsers import FormParser
-from rest_framework.views import APIView
-
-from core.lib.threshold import annotate_online, get_chans, get_users
-
-logger = logging.getLogger(__name__)
-
-
-class ThresholdChans(LoginRequiredMixin, APIView):
-    parser_classes = [FormParser]
-    plan_name = "drilldown"
-
-    def post(self, request):
-        if not request.user.has_plan(self.plan_name):
-            return JsonResponse({"success": False})
-        if "net" not in request.data:
-            return JsonResponse({"success": False})
-        if "query" not in request.data:
-            return JsonResponse({"success": False})
-        net = request.data["net"]
-        query = request.data["query"]
-        channels = get_chans(net, [query])
-        if not channels:
-            return HttpResponse("")
-        channels_human = ", ".join(channels)
-        return HttpResponse(channels_human)
-
-
-class ThresholdUsers(LoginRequiredMixin, APIView):
-    parser_classes = [FormParser]
-    plan_name = "drilldown"
-
-    def post(self, request):
-        if not request.user.has_plan(self.plan_name):
-            return JsonResponse({"success": False})
-        if "net" not in request.data:
-            return JsonResponse({"success": False})
-        if "query" not in request.data:
-            return JsonResponse({"success": False})
-        net = request.data["net"]
-        query = request.data["query"]
-        users = get_users(net, [query])
-        if not users:
-            return HttpResponse("")
-        users_human = ", ".join(users)
-        return HttpResponse(users_human)
-
-
-class ThresholdOnline(LoginRequiredMixin, APIView):
-    parser_classes = [FormParser]
-    plan_name = "drilldown"
-
-    def post(self, request):
-        if not request.user.has_plan(self.plan_name):
-            return JsonResponse({"success": False})
-        if "net" not in request.data:
-            return JsonResponse({"success": False})
-        if "query" not in request.data:
-            return JsonResponse({"success": False})
-        net = request.data["net"]
-        query = request.data["query"]
-        online_info = annotate_online(net, query)
-        return JsonResponse(online_info)

core/db/__init__.py

@@ -0,0 +1,324 @@
+import random
+import string
+import time
+from datetime import datetime
+from math import floor, log10
+
+import orjson
+from django.conf import settings
+from siphashc import siphash
+
+from core import r
+from core.db.processing import annotate_results
+from core.util import logs
+from core.views import helpers
+
+
+class StorageBackend(object):
+    def __init__(self, name):
+        self.log = logs.get_logger(name)
+        self.log.info(f"Initialising storage backend {name}")
+
+        self.initialise_caching()
+        self.initialise()
+
+    def initialise(self, **kwargs):
+        raise NotImplementedError
+
+    def initialise_caching(self):
+        hash_key = r.get("cache_hash_key")
+        if not hash_key:
+            letters = string.ascii_lowercase
+            hash_key = "".join(random.choice(letters) for i in range(16))
+            self.log.debug(f"Created new hash key: {hash_key}")
+            r.set("cache_hash_key", hash_key)
+        else:
+            hash_key = hash_key.decode("ascii")
+            self.log.debug(f"Decoded hash key: {hash_key}")
+        self.hash_key = hash_key
+
+    def construct_query(self, **kwargs):
+        raise NotImplementedError
+
+    def run_query(self, **kwargs):
+        raise NotImplementedError
+
+    def parse_size(self, query_params, sizes):
+        if "size" in query_params:
+            size = query_params["size"]
+            if size not in sizes:
+                message = "Size is not permitted"
+                message_class = "danger"
+                return {"message": message, "class": message_class}
+            size = int(size)
+        else:
+            size = 15
+
+        return size
+
+    def parse_index(self, user, query_params):
+        if "index" in query_params:
+            index = query_params["index"]
+            if index == "main":
+                index = settings.INDEX_MAIN
+            else:
+                if not user.has_perm(f"core.index_{index}"):
+                    message = "Not permitted to search by this index"
+                    message_class = "danger"
+                    return {
+                        "message": message,
+                        "class": message_class,
+                    }
+                if index == "meta":
+                    index = settings.INDEX_META
+                elif index == "internal":
+                    index = settings.INDEX_INT
+                elif index == "restricted":
+                    if not user.has_perm("core.restricted_sources"):
+                        message = "Not permitted to search by this index"
+                        message_class = "danger"
+                        return {
+                            "message": message,
+                            "class": message_class,
+                        }
+                    index = settings.INDEX_RESTRICTED
+                else:
+                    message = "Index is not valid."
+                    message_class = "danger"
+                    return {
+                        "message": message,
+                        "class": message_class,
+                    }
+        else:
+            index = settings.INDEX_MAIN
+        return index
+
+    def parse_query(self, query_params, tags, size, index, custom_query, add_bool):
+        query_created = False
+        if "query" in query_params:
+            query = query_params["query"]
+            search_query = self.construct_query(query, size, index)
+            query_created = True
+        else:
+            if custom_query:
+                search_query = custom_query
+            else:
+                search_query = self.construct_query(None, size, index, blank=True)
+
+        if tags:
+            # Get a blank search query
+            if not query_created:
+                search_query = self.construct_query(None, size, index, blank=True)
+                query_created = True
+            for tagname, tagvalue in tags.items():
+                add_bool.append({tagname: tagvalue})
+
+        valid = self.check_valid_query(query_params, custom_query)
+        if isinstance(valid, dict):
+            return valid
+
+        return search_query
+
+    def check_valid_query(self, query_params, custom_query):
+        required_any = ["query", "tags"]
+        if not any([field in query_params.keys() for field in required_any]):
+            if not custom_query:
+                message = "Empty query!"
+                message_class = "warning"
+                return {"message": message, "class": message_class}
+
+    def parse_source(self, user, query_params):
+        if "source" in query_params:
+            source = query_params["source"]
+
+            if source in settings.SOURCES_RESTRICTED:
+                if not user.has_perm("core.restricted_sources"):
+                    message = "Access denied"
+                    message_class = "danger"
+                    return {"message": message, "class": message_class}
+            elif source not in settings.MAIN_SOURCES:
+                message = "Invalid source"
+                message_class = "danger"
+                return {"message": message, "class": message_class}
+
+            if source == "all":
+                source = None  # the next block will populate it
+
+        if source:
+            sources = [source]
+        else:
+            sources = list(settings.MAIN_SOURCES)
+            if user.has_perm("core.restricted_sources"):
+                for source_iter in settings.SOURCES_RESTRICTED:
+                    sources.append(source_iter)
+
+        if "all" in sources:
+            sources.remove("all")
+
+        return sources
+
+    def parse_sort(self, query_params):
+        sort = None
+        if "sorting" in query_params:
+            sorting = query_params["sorting"]
+            if sorting not in ("asc", "desc", "none"):
+                message = "Invalid sort"
+                message_class = "danger"
+                return {"message": message, "class": message_class}
+            if sorting == "asc":
+                sort = "ascending"
+            elif sorting == "desc":
+                sort = "descending"
+        return sort
+
+    def parse_date_time(self, query_params):
+        if set({"from_date", "to_date", "from_time", "to_time"}).issubset(
+            query_params.keys()
+        ):
+            from_ts = f"{query_params['from_date']}T{query_params['from_time']}Z"
+            to_ts = f"{query_params['to_date']}T{query_params['to_time']}Z"
+            from_ts = datetime.strptime(from_ts, "%Y-%m-%dT%H:%MZ")
+            to_ts = datetime.strptime(to_ts, "%Y-%m-%dT%H:%MZ")
+
+            return (from_ts, to_ts)
+        return (None, None)
+
+    def parse_sentiment(self, query_params):
+        sentiment = None
+        if "check_sentiment" in query_params:
+            if "sentiment_method" not in query_params:
+                message = "No sentiment method"
+                message_class = "danger"
+                return {"message": message, "class": message_class}
+            if "sentiment" in query_params:
+                sentiment = query_params["sentiment"]
+                try:
+                    sentiment = float(sentiment)
+                except ValueError:
+                    message = "Sentiment is not a float"
+                    message_class = "danger"
+                    return {"message": message, "class": message_class}
+            sentiment_method = query_params["sentiment_method"]
+
+            return (sentiment_method, sentiment)
+
+    def filter_blacklisted(self, user, response):
+        """
+        Low level filter to take the raw search response and remove
+        objects from it we want to keep secret.
+        Does not return, the object is mutated in place.
+        """
+        response["redacted"] = 0
+        response["exemption"] = None
+        if user.is_superuser:
+            response["exemption"] = True
+        # is_anonymous = isinstance(user, AnonymousUser)
+        # For every hit from ES
+        for index, item in enumerate(list(response["hits"]["hits"])):
+            # For every blacklisted type
+            for blacklisted_type in settings.OPENSEARCH_BLACKLISTED.keys():
+                # Check this field we are matching exists
+                if "_source" in item.keys():
+                    data_index = "_source"
+                elif "fields" in item.keys():
+                    data_index = "fields"
+                else:
+                    return False
+                if blacklisted_type in item[data_index].keys():
+                    content = item[data_index][blacklisted_type]
+                    # For every item in the blacklisted array for the type
+                    for blacklisted_item in settings.OPENSEARCH_BLACKLISTED[
+                        blacklisted_type
+                    ]:
+                        if blacklisted_item == str(content):
+                            # Remove the item
+                            if item in response["hits"]["hits"]:
+                                # Let the UI know something was redacted
+                                if (
+                                    "exemption"
+                                    not in response["hits"]["hits"][index][data_index]
+                                ):
+                                    response["redacted"] += 1
+                                # Anonymous
+                                if user.is_anonymous:
+                                    # Just set it to none so the index is not off
+                                    response["hits"]["hits"][index] = None
+                                else:
+                                    if not user.has_perm("core.bypass_blacklist"):
+                                        response["hits"]["hits"][index] = None
+                                    else:
+                                        response["hits"]["hits"][index][data_index][
+                                            "exemption"
+                                        ] = True
+
+        # Actually get rid of all the things we set to None
+        response["hits"]["hits"] = [hit for hit in response["hits"]["hits"] if hit]
+
+    def query(self, user, search_query):
+        # For time tracking
+        start = time.process_time()
+        if settings.CACHE:
+            # Sort the keys so the hash is the same
+            query_normalised = orjson.dumps(search_query, option=orjson.OPT_SORT_KEYS)
+            hash = siphash(self.hash_key, query_normalised)
+            cache_hit = r.get(f"query_cache.{user.id}.{hash}")
+            if cache_hit:
+                response = orjson.loads(cache_hit)
+                print("CACHE HIT", response)
+
+                time_took = (time.process_time() - start) * 1000
+                # Round to 3 significant figures
+                time_took_rounded = round(
+                    time_took, 3 - int(floor(log10(abs(time_took)))) - 1
+                )
+                return {
+                    "object_list": response,
+                    "took": time_took_rounded,
+                    "cache": True,
+                }
+        response = self.run_query(user, search_query)
+        if "error" in response:
+            if "errorMessage" in response:
+                context = {
+                    "message": response["errorMessage"],
+                    "class": "danger",
+                }
+                return context
+            else:
+                return response
+        # response = response.to_dict()
+        # print("RESP", response)
+        if "took" in response:
+            if response["took"] is None:
+                return None
+        self.filter_blacklisted(user, response)
+
+        # Parse the response
+        response_parsed = self.parse(response)
+
+        # Write cache
+        if settings.CACHE:
+            to_write_cache = orjson.dumps(response_parsed)
+            r.set(f"query_cache.{user.id}.{hash}", to_write_cache)
+            r.expire(f"query_cache.{user.id}.{hash}", settings.CACHE_TIMEOUT)
+
+        time_took = (time.process_time() - start) * 1000
+        # Round to 3 significant figures
+        time_took_rounded = round(time_took, 3 - int(floor(log10(abs(time_took)))) - 1)
+        return {"object_list": response_parsed, "took": time_took_rounded}
+
+    def query_results(self, **kwargs):
+        raise NotImplementedError
+
+    def process_results(self, response, **kwargs):
+        if kwargs.get("annotate"):
+            annotate_results(response)
+        if kwargs.get("dedup"):
+            response = response[::-1]
+        if kwargs.get("dedup"):
+            if not kwargs.get("dedup_fields"):
+                dedup_fields = ["msg", "nick", "ident", "host", "net", "channel"]
+            response = helpers.dedup_list(response, dedup_fields)
+
+    def parse(self, response):
+        raise NotImplementedError

core/db/druid.py

@@ -0,0 +1,217 @@
+import logging
+
+import orjson
+import requests
+from django.conf import settings
+
+from core.db import StorageBackend
+from core.db.processing import parse_druid
+from core.views import helpers
+
+logger = logging.getLogger(__name__)
+
+
+class DruidBackend(StorageBackend):
+    def __init__(self):
+        super().__init__("Druid")
+
+    def initialise(self, **kwargs):
+        # self.client = PyDruid("http://broker:8082", "druid/v2")
+        pass  # we use requests
+
+    def construct_query(self, query, size, index, blank=False):
+        search_query = {
+            "limit": size,
+            "queryType": "scan",
+            "dataSource": index,
+            "intervals": ["1999-01-01/2999-01-01"],
+        }
+
+        base_filter = {
+            "type": "and",
+            "fields": [],
+        }
+        to_add = {
+            "type": "search",
+            "dimension": "msg",
+            "query": {
+                "type": "insensitive_contains",
+                "value": query,
+            },
+        }
+
+        if blank:
+            return search_query
+        else:
+            search_query["filter"] = base_filter
+            search_query["filter"]["fields"].append(to_add)
+            return search_query
+
+    def parse(self, response):
+        parsed = parse_druid(response)
+        print("PARSE LEN", len(parsed))
+        return parsed
+
+    def run_query(self, user, search_query):
+        ss = orjson.dumps(search_query, option=orjson.OPT_INDENT_2)
+        ss = ss.decode()
+        print(ss)
+        response = requests.post("http://broker:8082/druid/v2", json=search_query)
+        response = orjson.loads(response.text)
+        print("RESPONSE LEN", len(response))
+        # ss = orjson.dumps(response, option=orjson.OPT_INDENT_2)
+        # ss = ss.decode()
+        # print(ss)
+        return response
+
+    def filter_blacklisted(self, user, response):
+        pass
+
+    def query_results(
+        self,
+        request,
+        query_params,
+        size=None,
+        annotate=True,
+        custom_query=False,
+        reverse=False,
+        dedup=False,
+        dedup_fields=None,
+        tags=None,
+    ):
+        add_bool = []
+        add_in = {}
+
+        helpers.add_defaults(query_params)
+
+        # Now, run the helpers for SIQTSRSS/ADR
+        # S - Size
+        # I - Index
+        # Q - Query
+        # T - Tags
+        # S - Source
+        # R - Ranges
+        # S - Sort
+        # S - Sentiment
+        # A - Annotate
+        # D - Dedup
+        # R - Reverse
+
+        # S - Size
+        if request.user.is_anonymous:
+            sizes = settings.MAIN_SIZES_ANON
+        else:
+            sizes = settings.MAIN_SIZES
+        if not size:
+            size = self.parse_size(query_params, sizes)
+            if isinstance(size, dict):
+                return size
+
+        # I - Index
+        index = self.parse_index(request.user, query_params)
+        if isinstance(index, dict):
+            return index
+
+        # Q/T - Query/Tags
+        search_query = self.parse_query(
+            query_params, tags, size, index, custom_query, add_bool
+        )
+        # Query should be a dict, so check if it contains message here
+        if "message" in search_query:
+            return search_query
+
+        # S - Sources
+        sources = self.parse_source(request.user, query_params)
+        if isinstance(sources, dict):
+            return sources
+        total_count = len(sources)
+        total_sources = len(settings.MAIN_SOURCES) + len(settings.SOURCES_RESTRICTED)
+        if total_count != total_sources:
+            add_in["src"] = sources
+
+        # R - Ranges
+        from_ts, to_ts = self.parse_date_time(query_params)
+        if from_ts:
+            addendum = f"{from_ts}/{to_ts}"
+            search_query["intervals"] = [addendum]
+
+        # S - Sort
+        sort = self.parse_sort(query_params)
+        if isinstance(sort, dict):
+            return sort
+        if sort:
+            search_query["order"] = sort
+
+        # S - Sentiment
+        sentiment_r = self.parse_sentiment(query_params)
+        if isinstance(sentiment_r, dict):
+            return sentiment_r
+        if sentiment_r:
+            sentiment_method, sentiment = sentiment_r
+            sentiment_query = {"type": "bound", "dimension": "sentiment"}
+            if sentiment_method == "below":
+                sentiment_query["upper"] = sentiment
+            elif sentiment_method == "above":
+                sentiment_query["lower"] = sentiment
+            elif sentiment_method == "exact":
+                sentiment_query["lower"] = sentiment
+                sentiment_query["upper"] = sentiment
+            elif sentiment_method == "nonzero":
+                sentiment_query["lower"] = -0.0001
+                sentiment_query["upper"] = 0.0001
+                sentiment_query["lowerStrict"] = True
+                sentiment_query["upperStrict"] = True
+            # add_bool.append(sentiment_query)
+            self.add_filter(search_query)
+            search_query["filter"]["fields"].append(sentiment_query)
+
+        # Add in the additional information we already populated
+        if add_bool:
+            self.add_type("and", search_query, add_bool)
+        if add_in:
+            self.add_in(search_query, add_in)
+
+        response = self.query(request.user, search_query)
+
+        # A/D/R - Annotate/Dedup/Reverse
+        self.process_results(
+            response,
+            annotate=annotate,
+            dedup=dedup,
+            dedup_fields=dedup_fields,
+            reverse=reverse,
+        )
+        # ss = orjson.dumps(list(response), option=orjson.OPT_INDENT_2)
+        # ss = ss.decode()
+        # print(ss)
+        # print("PARSED", results_parsed)
+        # return results_parsed
+        context = response
+        return context
+
+    def add_filter(self, search_query):
+        if "filter" not in search_query:
+            search_query["filter"] = {
+                "type": "and",
+                "fields": [],
+            }
+
+    def add_in(self, search_query, add_in):
+        self.add_filter(search_query)
+        for key, value in add_in.items():
+            to_add = {"type": "in", "dimension": key, "values": value}
+            search_query["filter"]["fields"].append(to_add)
+
+    def add_type(self, gate, search_query, add_bool):
+        top_level_bool = {"type": gate, "fields": []}
+        self.add_filter(search_query)
+        for item in add_bool:
+            for key, value in item.items():
+                to_add = {"type": "selector", "dimension": key, "value": value}
+                top_level_bool["fields"].append(to_add)
+
+        search_query["filter"]["fields"].append(top_level_bool)
+
+    def check_valid_query(self, query_params, custom_query):
+        # We can do blank queries with this data source
+        pass

core/db/manticore.py

@@ -0,0 +1,303 @@
+import logging
+from datetime import datetime
+from pprint import pprint
+
+import requests
+from django.conf import settings
+
+from core.db import StorageBackend
+from core.db.processing import annotate_results, parse_results
+from core.views import helpers
+
+logger = logging.getLogger(__name__)
+
+
+class ManticoreBackend(StorageBackend):
+    def __init__(self):
+        super().__init__("Manticore")
+
+    def initialise(self, **kwargs):
+        """
+        Initialise the Manticore client
+        """
+        pass  # we use requests
+
+    def construct_query(self, query, size, index, blank=False):
+        """
+        Accept some query parameters and construct an OpenSearch query.
+        """
+        if not size:
+            size = 5
+        query_base = {
+            "index": index,
+            "limit": size,
+            "query": {"bool": {"must": []}},
+        }
+        query_string = {
+            "query_string": query,
+        }
+        if not blank:
+            query_base["query"]["bool"]["must"].append(query_string)
+        return query_base
+
+    def run_query(self, client, user, search_query):
+        response = requests.post(
+            f"{settings.MANTICORE_URL}/json/search", json=search_query
+        )
+        return response
+
+    def query_results(
+        self,
+        request,
+        query_params,
+        size=None,
+        annotate=True,
+        custom_query=False,
+        reverse=False,
+        dedup=False,
+        dedup_fields=None,
+        tags=None,
+    ):
+        query = None
+        message = None
+        message_class = None
+        add_bool = []
+        add_top = []
+        add_top_negative = []
+        sort = None
+        query_created = False
+        source = None
+        helpers.add_defaults(query_params)
+        # Check size
+        if request.user.is_anonymous:
+            sizes = settings.MANTICORE_MAIN_SIZES_ANON
+        else:
+            sizes = settings.MANTICORE_MAIN_SIZES
+        if not size:
+            if "size" in query_params:
+                size = query_params["size"]
+                if size not in sizes:
+                    message = "Size is not permitted"
+                    message_class = "danger"
+                    return {"message": message, "class": message_class}
+                size = int(size)
+            else:
+                size = 20
+
+        # Check index
+        if "index" in query_params:
+            index = query_params["index"]
+            if index == "main":
+                index = settings.MANTICORE_INDEX_MAIN
+            else:
+                if not request.user.has_perm(f"core.index_{index}"):
+                    message = "Not permitted to search by this index"
+                    message_class = "danger"
+                    return {
+                        "message": message,
+                        "class": message_class,
+                    }
+                if index == "meta":
+                    index = settings.MANTICORE_INDEX_META
+                elif index == "internal":
+                    index = settings.MANTICORE_INDEX_INT
+                else:
+                    message = "Index is not valid."
+                    message_class = "danger"
+                    return {
+                        "message": message,
+                        "class": message_class,
+                    }
+        else:
+            index = settings.MANTICORE_INDEX_MAIN
+
+        # Create the search query
+        if "query" in query_params:
+            query = query_params["query"]
+            search_query = self.construct_query(query, size, index)
+            query_created = True
+        else:
+            if custom_query:
+                search_query = custom_query
+
+        if tags:
+            # Get a blank search query
+            if not query_created:
+                search_query = self.construct_query(None, size, index, blank=True)
+                query_created = True
+            for tagname, tagvalue in tags.items():
+                add_bool.append({tagname: tagvalue})
+
+        required_any = ["query_full", "query", "tags"]
+        if not any([field in query_params.keys() for field in required_any]):
+            if not custom_query:
+                message = "Empty query!"
+                message_class = "warning"
+                return {"message": message, "class": message_class}
+
+        # Check for a source
+        if "source" in query_params:
+            source = query_params["source"]
+
+            if source in settings.SOURCES_RESTRICTED:
+                if not request.user.has_perm("core.restricted_sources"):
+                    message = "Access denied"
+                    message_class = "danger"
+                    return {"message": message, "class": message_class}
+            elif source not in settings.MAIN_SOURCES:
+                message = "Invalid source"
+                message_class = "danger"
+                return {"message": message, "class": message_class}
+
+            if source == "all":
+                source = None  # the next block will populate it
+
+        if source:
+            sources = [source]
+        else:
+            sources = list(settings.MAIN_SOURCES)
+            if request.user.has_perm("core.restricted_sources"):
+                for source_iter in settings.SOURCES_RESTRICTED:
+                    sources.append(source_iter)
+
+        add_top_tmp = {"bool": {"should": []}}
+        total_count = 0
+        for source_iter in sources:
+            add_top_tmp["bool"]["should"].append({"equals": {"src": source_iter}})
+            total_count += 1
+        total_sources = len(settings.MAIN_SOURCES) + len(settings.SOURCES_RESTRICTED)
+        if not total_count == total_sources:
+            add_top.append(add_top_tmp)
+
+        # Date/time range
+        if set({"from_date", "to_date", "from_time", "to_time"}).issubset(
+            query_params.keys()
+        ):
+            from_ts = f"{query_params['from_date']}T{query_params['from_time']}Z"
+            to_ts = f"{query_params['to_date']}T{query_params['to_time']}Z"
+            from_ts = datetime.strptime(from_ts, "%Y-%m-%dT%H:%MZ")
+            to_ts = datetime.strptime(to_ts, "%Y-%m-%dT%H:%MZ")
+            from_ts = int(from_ts.timestamp())
+            to_ts = int(to_ts.timestamp())
+            range_query = {
+                "range": {
+                    "ts": {
+                        "gt": from_ts,
+                        "lt": to_ts,
+                    }
+                }
+            }
+            add_top.append(range_query)
+
+        # Sorting
+        if "sorting" in query_params:
+            sorting = query_params["sorting"]
+            if sorting not in ("asc", "desc", "none"):
+                message = "Invalid sort"
+                message_class = "danger"
+                return {"message": message, "class": message_class}
+            if sorting in ("asc", "desc"):
+                sort = [
+                    {
+                        "ts": {
+                            "order": sorting,
+                        }
+                    }
+                ]
+
+        # Sentiment handling
+        if "check_sentiment" in query_params:
+            if "sentiment_method" not in query_params:
+                message = "No sentiment method"
+                message_class = "danger"
+                return {"message": message, "class": message_class}
+            if "sentiment" in query_params:
+                sentiment = query_params["sentiment"]
+                try:
+                    sentiment = float(sentiment)
+                except ValueError:
+                    message = "Sentiment is not a float"
+                    message_class = "danger"
+                    return {"message": message, "class": message_class}
+            sentiment_method = query_params["sentiment_method"]
+            range_query_compare = {"range": {"sentiment": {}}}
+            range_query_precise = {
+                "match": {
+                    "sentiment": None,
+                }
+            }
+            if sentiment_method == "below":
+                range_query_compare["range"]["sentiment"]["lt"] = sentiment
+                add_top.append(range_query_compare)
+            elif sentiment_method == "above":
+                range_query_compare["range"]["sentiment"]["gt"] = sentiment
+                add_top.append(range_query_compare)
+            elif sentiment_method == "exact":
+                range_query_precise["match"]["sentiment"] = sentiment
+                add_top.append(range_query_precise)
+            elif sentiment_method == "nonzero":
+                range_query_precise["match"]["sentiment"] = 0
+                add_top_negative.append(range_query_precise)
+
+        if add_bool:
+            # if "bool" not in search_query["query"]:
+            #     search_query["query"]["bool"] = {}
+            # if "must" not in search_query["query"]["bool"]:
+            #     search_query["query"]["bool"] = {"must": []}
+
+            for item in add_bool:
+                search_query["query"]["bool"]["must"].append({"match": item})
+
+        if add_top:
+            for item in add_top:
+                search_query["query"]["bool"]["must"].append(item)
+        if add_top_negative:
+            for item in add_top_negative:
+                if "must_not" in search_query["query"]["bool"]:
+                    search_query["query"]["bool"]["must_not"].append(item)
+                else:
+                    search_query["query"]["bool"]["must_not"] = [item]
+        if sort:
+            search_query["sort"] = sort
+
+        pprint(search_query)
+        results = self.run_query(
+            self.client,
+            request.user,  # passed through run_main_query to filter_blacklisted
+            search_query,
+        )
+        if not results:
+            message = "Error running query"
+            message_class = "danger"
+            return {"message": message, "class": message_class}
+        # results = results.to_dict()
+        if "error" in results:
+            message = results["error"]
+            message_class = "danger"
+            return {"message": message, "class": message_class}
+        results_parsed = parse_results(results)
+        if annotate:
+            annotate_results(results_parsed)
+        if "dedup" in query_params:
+            if query_params["dedup"] == "on":
+                dedup = True
+            else:
+                dedup = False
+        else:
+            dedup = False
+
+        if reverse:
+            results_parsed = results_parsed[::-1]
+
+        if dedup:
+            if not dedup_fields:
+                dedup_fields = ["msg", "nick", "ident", "host", "net", "channel"]
+            results_parsed = helpers.dedup_list(results_parsed, dedup_fields)
+        context = {
+            "object_list": results_parsed,
+            "card": results["hits"]["total"],
+            "took": results["took"],
+        }
+        if "cache" in results:
+            context["cache"] = results["cache"]
+        return context

core/db/opensearch.py

@@ -0,0 +1,485 @@
+# from copy import deepcopy
+# from datetime import datetime, timedelta
+
+from django.conf import settings
+from opensearchpy import OpenSearch
+from opensearchpy.exceptions import NotFoundError, RequestError
+
+from core.db import StorageBackend
+
+# from json import dumps
+# pp = lambda x: print(dumps(x, indent=2))
+from core.db.processing import annotate_results, parse_results
+from core.views.helpers import dedup_list
+
+
+class OpensearchBackend(StorageBackend):
+    def __init__(self):
+        super().__init__("Opensearch")
+
+    def initialise(self, **kwargs):
+        """
+        Inititialise the OpenSearch API endpoint.
+        """
+        auth = (settings.OPENSEARCH_USERNAME, settings.OPENSEARCH_PASSWORD)
+        client = OpenSearch(
+            # fmt: off
+            hosts=[{"host": settings.OPENSEARCH_URL,
+                    "port": settings.OPENSEARCH_PORT}],
+            http_compress=False,  # enables gzip compression for request bodies
+            http_auth=auth,
+            # client_cert = client_cert_path,
+            # client_key = client_key_path,
+            use_ssl=settings.OPENSEARCH_TLS,
+            verify_certs=False,
+            ssl_assert_hostname=False,
+            ssl_show_warn=False,
+            # a_certs=ca_certs_path,
+        )
+        self.client = client
+
+    def construct_query(self, query, size, use_query_string=True, tokens=False):
+        """
+        Accept some query parameters and construct an OpenSearch query.
+        """
+        if not size:
+            size = 5
+        query_base = {
+            "size": size,
+            "query": {"bool": {"must": []}},
+        }
+        query_string = {
+            "query_string": {
+                "query": query,
+                # "fields": fields,
+                # "default_field": "msg",
+                # "type": "best_fields",
+                "fuzziness": "AUTO",
+                "fuzzy_transpositions": True,
+                "fuzzy_max_expansions": 50,
+                "fuzzy_prefix_length": 0,
+                # "minimum_should_match": 1,
+                "default_operator": "or",
+                "analyzer": "standard",
+                "lenient": True,
+                "boost": 1,
+                "allow_leading_wildcard": True,
+                # "enable_position_increments": False,
+                "phrase_slop": 3,
+                # "max_determinized_states": 10000,
+                "quote_field_suffix": "",
+                "quote_analyzer": "standard",
+                "analyze_wildcard": False,
+                "auto_generate_synonyms_phrase_query": True,
+            }
+        }
+        query_tokens = {
+            "simple_query_string": {
+                # "tokens": query,
+                "query": query,
+                "fields": ["tokens"],
+                "flags": "ALL",
+                "fuzzy_transpositions": True,
+                "fuzzy_max_expansions": 50,
+                "fuzzy_prefix_length": 0,
+                "default_operator": "and",
+                "analyzer": "standard",
+                "lenient": True,
+                "boost": 1,
+                "quote_field_suffix": "",
+                "analyze_wildcard": False,
+                "auto_generate_synonyms_phrase_query": False,
+            }
+        }
+        if tokens:
+            query_base["query"]["bool"]["must"].append(query_tokens)
+            # query["query"]["bool"]["must"].append(query_string)
+            # query["query"]["bool"]["must"][0]["query_string"]["fields"] = ["tokens"]
+        elif use_query_string:
+            query_base["query"]["bool"]["must"].append(query_string)
+        return query_base
+
+    def run_query(self, client, user, query, custom_query=False, index=None, size=None):
+        """
+        Low level helper to run an ES query.
+        Accept a user to pass it to the filter, so we can
+        avoid filtering for superusers.
+        Accept fields and size, for the fields we want to match and the
+        number of results to return.
+        """
+        if not index:
+            index = settings.INDEX_MAIN
+        if custom_query:
+            search_query = query
+        else:
+            search_query = self.construct_query(query, size)
+        try:
+            response = client.search(body=search_query, index=index)
+        except RequestError as err:
+            print("OpenSearch error", err)
+            return err
+        except NotFoundError as err:
+            print("OpenSearch error", err)
+            return err
+        return response
+
+    def query_results(
+        self,
+        request,
+        query_params,
+        size=None,
+        annotate=True,
+        custom_query=False,
+        reverse=False,
+        dedup=False,
+        dedup_fields=None,
+        lookup_hashes=True,
+        tags=None,
+    ):
+        """
+        API helper to alter the OpenSearch return format into something
+        a bit better to parse.
+        Accept a HTTP request object. Run the query, and annotate the
+        results with the other data we have.
+        """
+        # is_anonymous = isinstance(request.user, AnonymousUser)
+        query = None
+        message = None
+        message_class = None
+        add_bool = []
+        add_top = []
+        add_top_negative = []
+        sort = None
+        query_created = False
+
+        # Lookup the hash values but don't disclose them to the user
+        # denied = []
+        # if lookup_hashes:
+        #     if settings.HASHING:
+        #         query_params = deepcopy(query_params)
+        #         denied_q = hash_lookup(request.user, query_params)
+        #         denied.extend(denied_q)
+        #         if tags:
+        #             denied_t = hash_lookup(request.user, tags, query_params)
+        #             denied.extend(denied_t)
+
+        # message = "Permission denied: "
+        # for x in denied:
+        #     if isinstance(x, SearchDenied):
+        #         message += f"Search({x.key}: {x.value}) "
+        #     elif isinstance(x, LookupDenied):
+        #         message += f"Lookup({x.key}: {x.value}) "
+        # if denied:
+        #     # message = [f"{i}" for i in message]
+        #     # message = "\n".join(message)
+        #     message_class = "danger"
+        #     return {"message": message, "class": message_class}
+
+        if request.user.is_anonymous:
+            sizes = settings.MAIN_SIZES_ANON
+        else:
+            sizes = settings.MAIN_SIZES
+        if not size:
+            if "size" in query_params:
+                size = query_params["size"]
+                if size not in sizes:
+                    message = "Size is not permitted"
+                    message_class = "danger"
+                    return {"message": message, "class": message_class}
+            else:
+                size = 20
+        source = None
+        if "source" in query_params:
+            source = query_params["source"]
+
+            if source in settings.SOURCES_RESTRICTED:
+                if not request.user.has_perm("core.restricted_sources"):
+                    message = "Access denied"
+                    message_class = "danger"
+                    return {"message": message, "class": message_class}
+            elif source not in settings.MAIN_SOURCES:
+                message = "Invalid source"
+                message_class = "danger"
+                return {"message": message, "class": message_class}
+
+            if source == "all":
+                source = None  # the next block will populate it
+
+        if source:
+            sources = [source]
+        else:
+            sources = settings.MAIN_SOURCES
+            if request.user.has_perm("core.restricted_sources"):
+                for source_iter in settings.SOURCES_RESTRICTED:
+                    sources.append(source_iter)
+
+        add_top_tmp = {"bool": {"should": []}}
+        for source_iter in sources:
+            add_top_tmp["bool"]["should"].append({"match_phrase": {"src": source_iter}})
+        add_top.append(add_top_tmp)
+
+        # date_query = False
+        if set({"from_date", "to_date", "from_time", "to_time"}).issubset(
+            query_params.keys()
+        ):
+            from_ts = f"{query_params['from_date']}T{query_params['from_time']}Z"
+            to_ts = f"{query_params['to_date']}T{query_params['to_time']}Z"
+            range_query = {
+                "range": {
+                    "ts": {
+                        "gt": from_ts,
+                        "lt": to_ts,
+                    }
+                }
+            }
+            add_top.append(range_query)
+
+        # if date_query:
+        #     if settings.DELAY_RESULTS:
+        #         if source not in settings.SAFE_SOURCES:
+        #             if request.user.has_perm("core.bypass_delay"):
+        #                 add_top.append(range_query)
+        #             else:
+        #                 delay_as_ts = datetime.now() - timedelta(
+        #                     days=settings.DELAY_DURATION
+        #                 )
+        #                 lt_as_ts = datetime.strptime(
+        #                     range_query["range"]["ts"]["lt"], "%Y-%m-%dT%H:%MZ"
+        #                 )
+        #                 if lt_as_ts > delay_as_ts:
+        #                     range_query["range"]["ts"][
+        #                         "lt"
+        #                     ] = f"now-{settings.DELAY_DURATION}d"
+        #                 add_top.append(range_query)
+        #     else:
+        #         add_top.append(range_query)
+        # else:
+        #     if settings.DELAY_RESULTS:
+        #         if source not in settings.SAFE_SOURCES:
+        #             if not request.user.has_perm("core.bypass_delay"):
+        #                 range_query = {
+        #                     "range": {
+        #                         "ts": {
+        #                             # "gt": ,
+        #                             "lt": f"now-{settings.DELAY_DURATION}d",
+        #                         }
+        #                     }
+        #                 }
+        #                 add_top.append(range_query)
+
+        if "sorting" in query_params:
+            sorting = query_params["sorting"]
+            if sorting not in ("asc", "desc", "none"):
+                message = "Invalid sort"
+                message_class = "danger"
+                return {"message": message, "class": message_class}
+            if sorting in ("asc", "desc"):
+                sort = [
+                    {
+                        "ts": {
+                            "order": sorting,
+                        }
+                    }
+                ]
+
+        if "check_sentiment" in query_params:
+            if "sentiment_method" not in query_params:
+                message = "No sentiment method"
+                message_class = "danger"
+                return {"message": message, "class": message_class}
+            if "sentiment" in query_params:
+                sentiment = query_params["sentiment"]
+                try:
+                    sentiment = float(sentiment)
+                except ValueError:
+                    message = "Sentiment is not a float"
+                    message_class = "danger"
+                    return {"message": message, "class": message_class}
+            sentiment_method = query_params["sentiment_method"]
+            range_query_compare = {"range": {"sentiment": {}}}
+            range_query_precise = {
+                "match": {
+                    "sentiment": None,
+                }
+            }
+            if sentiment_method == "below":
+                range_query_compare["range"]["sentiment"]["lt"] = sentiment
+                add_top.append(range_query_compare)
+            elif sentiment_method == "above":
+                range_query_compare["range"]["sentiment"]["gt"] = sentiment
+                add_top.append(range_query_compare)
+            elif sentiment_method == "exact":
+                range_query_precise["match"]["sentiment"] = sentiment
+                add_top.append(range_query_precise)
+            elif sentiment_method == "nonzero":
+                range_query_precise["match"]["sentiment"] = 0
+                add_top_negative.append(range_query_precise)
+
+        # Only one of query or query_full can be active at once
+        # We prefer query because it's simpler
+        if "query" in query_params:
+            query = query_params["query"]
+            search_query = self.construct_query(query, size, tokens=True)
+            query_created = True
+        elif "query_full" in query_params:
+            query_full = query_params["query_full"]
+            # if request.user.has_perm("core.query_search"):
+            search_query = self.construct_query(query_full, size)
+            query_created = True
+            # else:
+            #     message = "You cannot search by query string"
+            #     message_class = "danger"
+            #     return {"message": message, "class": message_class}
+        else:
+            if custom_query:
+                search_query = custom_query
+
+        if tags:
+            # Get a blank search query
+            if not query_created:
+                search_query = self.construct_query(None, size, use_query_string=False)
+                query_created = True
+            for tagname, tagvalue in tags.items():
+                add_bool.append({tagname: tagvalue})
+
+        required_any = ["query_full", "query", "tags"]
+        if not any([field in query_params.keys() for field in required_any]):
+            if not custom_query:
+                message = "Empty query!"
+                message_class = "warning"
+                return {"message": message, "class": message_class}
+
+        if add_bool:
+            # if "bool" not in search_query["query"]:
+            #     search_query["query"]["bool"] = {}
+            # if "must" not in search_query["query"]["bool"]:
+            #     search_query["query"]["bool"] = {"must": []}
+
+            for item in add_bool:
+                search_query["query"]["bool"]["must"].append({"match_phrase": item})
+        if add_top:
+            for item in add_top:
+                search_query["query"]["bool"]["must"].append(item)
+        if add_top_negative:
+            for item in add_top_negative:
+                if "must_not" in search_query["query"]["bool"]:
+                    search_query["query"]["bool"]["must_not"].append(item)
+                else:
+                    search_query["query"]["bool"]["must_not"] = [item]
+        if sort:
+            search_query["sort"] = sort
+
+        if "index" in query_params:
+            index = query_params["index"]
+            if index == "main":
+                index = settings.INDEX_MAIN
+            else:
+                if not request.user.has_perm(f"core.index_{index}"):
+                    message = "Not permitted to search by this index"
+                    message_class = "danger"
+                    return {
+                        "message": message,
+                        "class": message_class,
+                    }
+                if index == "meta":
+                    index = settings.INDEX_META
+                elif index == "internal":
+                    index = settings.INDEX_INT
+                else:
+                    message = "Index is not valid."
+                    message_class = "danger"
+                    return {
+                        "message": message,
+                        "class": message_class,
+                    }
+
+        else:
+            index = settings.INDEX_MAIN
+
+        results = self.query(
+            request.user,  # passed through run_main_query to filter_blacklisted
+            search_query,
+            custom_query=True,
+            index=index,
+            size=size,
+        )
+        if not results:
+            return False
+        if isinstance(results, Exception):
+            message = f"Error: {results.info['error']['root_cause'][0]['type']}"
+            message_class = "danger"
+            return {"message": message, "class": message_class}
+        if len(results["hits"]["hits"]) == 0:
+            message = "No results."
+            message_class = "danger"
+            return {"message": message, "class": message_class}
+
+        results_parsed = parse_results(results)
+
+        if annotate:
+            annotate_results(results_parsed)
+        if "dedup" in query_params:
+            if query_params["dedup"] == "on":
+                dedup = True
+            else:
+                dedup = False
+        else:
+            dedup = False
+
+        if reverse:
+            results_parsed = results_parsed[::-1]
+
+        if dedup:
+            if not dedup_fields:
+                dedup_fields = ["msg", "nick", "ident", "host", "net", "channel"]
+            results_parsed = dedup_list(results_parsed, dedup_fields)
+
+        # if source not in settings.SAFE_SOURCES:
+        #     if settings.ENCRYPTION:
+        #         encrypt_list(request.user, results_parsed, settings.ENCRYPTION_KEY)
+
+        #     if settings.HASHING:
+        #         hash_list(request.user, results_parsed)
+
+        #     if settings.OBFUSCATION:
+        #         obfuscate_list(request.user, results_parsed)
+
+        #     if settings.RANDOMISATION:
+        #         randomise_list(request.user, results_parsed)
+
+        # process_list(results)
+
+        # IMPORTANT! - DO NOT PASS query_params to the user!
+        context = {
+            "object_list": results_parsed,
+            "card": results["hits"]["total"]["value"],
+            "took": results["took"],
+        }
+        if "redacted" in results:
+            context["redacted"] = results["redacted"]
+        if "exemption" in results:
+            context["exemption"] = results["exemption"]
+        if query:
+            context["query"] = query
+        # if settings.DELAY_RESULTS:
+        #     if source not in settings.SAFE_SOURCES:
+        #         if not request.user.has_perm("core.bypass_delay"):
+        #             context["delay"] = settings.DELAY_DURATION
+        # if settings.RANDOMISATION:
+        #     if source not in settings.SAFE_SOURCES:
+        #         if not request.user.has_perm("core.bypass_randomisation"):
+        #             context["randomised"] = True
+        return context
+
+    def query_single_result(self, request, query_params):
+        context = self.query_results(request, query_params, size=100)
+
+        if not context:
+            return {"message": "Failed to run query", "message_class": "danger"}
+        if "message" in context:
+            return context
+        dedup_set = {item["nick"] for item in context["object_list"]}
+        if dedup_set:
+            context["item"] = context["object_list"][0]
+
+        return context

core/db/processing.py

@@ -0,0 +1,124 @@
+from datetime import datetime
+
+from core.lib.threshold import annotate_num_chans, annotate_num_users, annotate_online
+
+
+def annotate_results(results_parsed):
+    """
+    Accept a list of dict objects, search for the number of channels and users.
+    Add them to the object.
+    Mutate it in place. Does not return anything.
+    """
+    # Figure out items with net (not discord)
+    nets = set()
+    for x in results_parsed:
+        if "net" in x:
+            nets.add(x["net"])
+
+    for net in nets:
+        # Annotate the online attribute from Threshold
+        nicks = list(
+            set(
+                [
+                    x["nick"]
+                    for x in results_parsed
+                    if {"nick", "src", "net"}.issubset(x)
+                    and x["src"] == "irc"
+                    and x["net"] == net
+                ]
+            )
+        )
+        channels = list(
+            set(
+                [
+                    x["channel"]
+                    for x in results_parsed
+                    if {"channel", "src", "net"}.issubset(x)
+                    and x["src"] == "irc"
+                    and x["net"] == net
+                ]
+            )
+        )
+        online_info = annotate_online(net, nicks)
+        # Annotate the number of users in the channel
+        num_users = annotate_num_users(net, channels)
+        # Annotate the number channels the user is on
+        num_chans = annotate_num_chans(net, nicks)
+        for item in results_parsed:
+            if "net" in item:
+                if item["net"] == net:
+                    if "nick" in item:
+                        if item["nick"] in online_info:
+                            item["online"] = online_info[item["nick"]]
+                    if "channel" in item:
+                        if item["channel"] in num_users:
+                            item["num_users"] = num_users[item["channel"]]
+                    if "nick" in item:
+                        if item["nick"] in num_chans:
+                            item["num_chans"] = num_chans[item["nick"]]
+
+
+def parse_results(results):
+    results_parsed = []
+    stringify = ["host", "channel"]
+    if "hits" in results.keys():
+        if "hits" in results["hits"]:
+            for item in results["hits"]["hits"]:
+                if "_source" in item.keys():
+                    data_index = "_source"
+                elif "fields" in item.keys():
+                    data_index = "fields"
+                else:
+                    return False
+                element = item[data_index]
+                for field in stringify:
+                    if field in element:
+                        element[field] = str(element[field])
+                # Why are fields in lists...
+                if data_index == "fields":
+                    element = {k: v[0] for k, v in element.items() if len(v)}
+                element["id"] = item["_id"]
+
+                # Remove empty values
+                for field in list(element.keys()):
+                    if element[field] == "":
+                        del element[field]
+
+                # Split the timestamp into date and time
+                if "ts" not in element:
+                    if "time" in element:  # will fix data later
+                        ts = element["time"]
+                        del element["time"]
+                        element["ts"] = ts
+                if "ts" in element:
+                    if isinstance(element["ts"], str):
+                        ts = element["ts"]
+                    else:
+                        ts = datetime.utcfromtimestamp(element["ts"]).strftime(
+                            "%Y-%m-%dT%H:%M:%S"
+                        )
+                    ts_spl = ts.split("T")
+                    date = ts_spl[0]
+                    time = ts_spl[1]
+                    element["date"] = date
+                    if "." in time:
+                        time_spl = time.split(".")
+                        if len(time_spl) == 2:
+                            element["time"] = time.split(".")[0]
+                        else:
+                            element["time"] = time
+                    else:
+                        element["time"] = time
+                results_parsed.append(element)
+    return results_parsed
+
+
+def parse_druid(response):
+    results_parsed = []
+    for item in response:
+        if "events" in item:
+            for event in item["events"]:
+                results_parsed.append(event)
+        else:
+            raise Exception(f"events not in item {item}")
+    return results_parsed

core/db/storage.py

@@ -0,0 +1,21 @@
+from django.conf import settings
+
+
+def get_db():
+    if settings.DB_BACKEND == "DRUID":
+        from core.db.druid import DruidBackend
+
+        return DruidBackend()
+    elif settings.DB_BACKEND == "OPENSEARCH":
+        from core.db.opensearch import OpensearchBackend
+
+        return OpensearchBackend()
+    elif settings.DB_BACKEND == "MANTICORE":
+        from core.db.manticore import ManticoreBackend
+
+        return ManticoreBackend()
+    else:
+        raise Exception("Invalid DB backend")
+
+
+db = get_db()

core/lib/context.py

@@ -0,0 +1,87 @@
+def construct_query(index, net, channel, src, num, size, type=None, nicks=None):
+    # Get the initial query
+    extra_must = []
+    extra_should = []
+    extra_should2 = []
+    if num:
+        extra_must.append({"equals": {"num": num}})
+    if net:
+        extra_must.append({"match_phrase": {"net": net}})
+    if channel:
+        extra_must.append({"match": {"channel": channel}})
+    if nicks:
+        for nick in nicks:
+            extra_should2.append({"match": {"nick": nick}})
+    types = ["msg", "notice", "action", "kick", "topic", "mode"]
+    fields = [
+        "nick",
+        "ident",
+        "host",
+        "channel",
+        "ts",
+        "msg",
+        "type",
+        "net",
+        "src",
+        "tokens",
+    ]
+    if index == "internal":
+        fields.append("mtype")
+        if channel == "*status" or type == "znc":
+            if {"match": {"channel": channel}} in extra_must:
+                extra_must.remove({"match": {"channel": channel}})
+            extra_should2 = []
+            # Type is one of msg or notice
+            # extra_should.append({"match": {"mtype": "msg"}})
+            # extra_should.append({"match": {"mtype": "notice"}})
+            extra_should.append({"match": {"type": "znc"}})
+            extra_should.append({"match": {"type": "self"}})
+
+            extra_should2.append({"match": {"type": "znc"}})
+            extra_should2.append({"match": {"nick": channel}})
+        elif type == "auth":
+            if {"match": {"channel": channel}} in extra_must:
+                extra_must.remove({"match": {"channel": channel}})
+            extra_should2 = []
+            extra_should2.append({"match": {"nick": channel}})
+            # extra_should2.append({"match": {"mtype": "msg"}})
+            # extra_should2.append({"match": {"mtype": "notice"}})
+
+            extra_should.append({"match": {"type": "query"}})
+            extra_should2.append({"match": {"type": "self"}})
+            extra_should.append({"match": {"nick": channel}})
+        else:
+            for ctype in types:
+                extra_should.append({"equals": {"mtype": ctype}})
+    else:
+        for ctype in types:
+            extra_should.append({"match": {"type": ctype}})
+    query = {
+        "index": index,
+        "limit": size,
+        "query": {
+            "bool": {
+                "must": [
+                    # {"equals": {"src": src}},
+                    # {
+                    #     "bool": {
+                    #         "should": [*extra_should],
+                    #     }
+                    # },
+                    # {
+                    #     "bool": {
+                    #         "should": [*extra_should2],
+                    #     }
+                    # },
+                    *extra_must,
+                ]
+            }
+        },
+        "fields": fields,
+        # "_source": False,
+    }
+    if extra_should:
+        query["query"]["bool"]["must"].append({"bool": {"should": [*extra_should]}})
+    if extra_should2:
+        query["query"]["bool"]["must"].append({"bool": {"should": [*extra_should2]}})
+    return query

core/lib/manage/threshold.py

@@ -1,8 +1,3 @@
-import urllib.parse
-
-from django.conf import settings
-
-from core.lib.opensearch import client, run_main_query
 from core.lib.threshold import threshold_request
 
 
@@ -66,9 +61,8 @@ def get_irc_channels(net):
 
 
 def part_channel(net, channel):
-    channel = urllib.parse.quote(channel, safe="")
-    url = f"irc/network/{net}/channel/{channel}"
-    payload = {}
+    url = f"irc/network/{net}/channel"
+    payload = {"channel": channel}
     parted = threshold_request(url, payload, method="DELETE")
     if not parted:
         return {}
@@ -76,9 +70,8 @@ def part_channel(net, channel):
 
 
 def join_channel(net, channel):
-    channel = urllib.parse.quote(channel, safe="")
-    url = f"irc/network/{net}/channel/{channel}"
-    payload = {}
+    url = f"irc/network/{net}/channel"
+    payload = {"channel": channel}
     joined = threshold_request(url, payload, method="PUT")
     if not joined:
         return {}
@@ -166,30 +159,69 @@ def construct_alert_query():
     return query
 
 
-def get_irc_alerts(user):
-    query = construct_alert_query()
-    results = run_main_query(
-        client,
-        user,  # passed through run_main_query to filter_blacklisted
-        query,
-        custom_query=True,
-        index=settings.OPENSEARCH_INDEX_INT,
-    )
-    if not results:
-        return []
-    results_parsed = []
-    if "hits" in results.keys():
-        if "hits" in results["hits"]:
-            for item in results["hits"]["hits"]:
-                element = item["_source"]
-                element["id"] = item["_id"]
+def send_irc_message(net, num, channel, msg, nick=None):
+    url = f"irc/msg/{net}/{num}"
+    payload = {"msg": msg, "channel": channel}
+    if nick:
+        payload["nick"] = nick
+    messaged = threshold_request(url, payload, method="PUT")
+    return messaged
 
-                # Split the timestamp into date and time
-                ts = element["ts"]
-                ts_spl = ts.split("T")
-                date = ts_spl[0]
-                time = ts_spl[1]
-                element["date"] = date
-                element["time"] = time
-                results_parsed.append(element)
-    return results_parsed
+
+def get_irc_nick(net, num):
+    url = f"irc/nick/{net}/{num}"
+    payload = {}
+    nick = threshold_request(url, payload, method="GET")
+    return nick
+
+
+def get_irc_list_info(net):
+    url = f"irc/list/{net}"
+    payload = {}
+    listinfo = threshold_request(url, payload, method="GET")
+    return listinfo
+
+
+def irc_get_unreg(net=None):
+    if net:
+        url = f"irc/reg/{net}"
+    else:
+        url = "irc/reg"
+    payload = {}
+    unreg = threshold_request(url, payload, method="GET")
+    return unreg
+
+
+def irc_confirm_accounts(tokens):
+    url = "irc/reg"
+    payload = tokens
+    updated = threshold_request(url, payload, method="PUT")
+    return updated
+
+
+def irc_provision_relay(net, num):
+    url = f"irc/network/{net}/{num}/provision"
+    payload = {}
+    provisioned = threshold_request(url, payload, method="POST")
+    return provisioned
+
+
+def irc_enable_auth(net, num):
+    url = f"irc/network/{net}/{num}/auth"
+    payload = {}
+    enabled = threshold_request(url, payload, method="POST")
+    return enabled
+
+
+def irc_check_auth(data):
+    url = "irc/auth"
+    payload = data
+    updated = threshold_request(url, payload, method="POST")
+    return updated
+
+
+def get_irc_sinst(net):
+    url = f"irc/sinst/{net}"
+    payload = {}
+    authentity = threshold_request(url, payload, method="GET")
+    return authentity

core/lib/meta.py

@@ -3,7 +3,7 @@ from math import ceil
 from django.conf import settings
 from numpy import array_split
 
-from core.lib.opensearch import client, run_main_query
+from core.db.opensearch import client, run_main_query
 
 
 def construct_query(net, nicks):

core/lib/nicktrace.py

@@ -3,7 +3,7 @@ from math import ceil
 from django.conf import settings
 from numpy import array_split
 
-from core.lib.opensearch import client, run_main_query
+from core.lib.druid import client, run_main_query
 
 
 def construct_query(net, nicks):

core/lib/opensearch.py

@@ -1,360 +0,0 @@
-from django.conf import settings
-from opensearchpy import OpenSearch
-from opensearchpy.exceptions import RequestError
-
-from core.lib.threshold import annotate_num_chans, annotate_num_users, annotate_online
-
-
-def initialise_opensearch():
-    """
-    Inititialise the OpenSearch API endpoint.
-    """
-    auth = (settings.OPENSEARCH_USERNAME, settings.OPENSEARCH_PASSWORD)
-    client = OpenSearch(
-        # fmt: off
-        hosts=[{"host": settings.OPENSEARCH_URL,
-                "port": settings.OPENSEARCH_PORT}],
-        http_compress=False,  # enables gzip compression for request bodies
-        http_auth=auth,
-        # client_cert = client_cert_path,
-        # client_key = client_key_path,
-        use_ssl=settings.OPENSEARCH_TLS,
-        verify_certs=False,
-        ssl_assert_hostname=False,
-        ssl_show_warn=False,
-        # a_certs=ca_certs_path,
-    )
-    return client
-
-
-client = initialise_opensearch()
-
-
-def annotate_results(results_parsed):
-    """
-    Accept a list of dict objects, search for the number of channels and users.
-    Add them to the object.
-    Mutate it in place. Does not return anything.
-    """
-    # Figure out items with net (not discord)
-    nets = set()
-    for x in results_parsed:
-        if "net" in x:
-            nets.add(x["net"])
-
-    for net in nets:
-        # Annotate the online attribute from Threshold
-        nicks = [
-            x["nick"] for x in results_parsed if x["src"] == "irc" and x["net"] == net
-        ]
-        channels = [
-            x["channel"]
-            for x in results_parsed
-            if x["src"] == "irc" and x["net"] == net
-        ]
-        online_info = annotate_online(net, nicks)
-        # Annotate the number of users in the channel
-        num_users = annotate_num_users(net, channels)
-        # Annotate the number channels the user is on
-        num_chans = annotate_num_chans(net, nicks)
-        for item in results_parsed:
-            if "net" in item:
-                if item["net"] == net:
-                    if "nick" in item:
-                        if item["nick"] in online_info:
-                            item["online"] = online_info[item["nick"]]
-                    if "channel" in item:
-                        if item["channel"] in num_users:
-                            item["num_users"] = num_users[item["channel"]]
-                    if "nick" in item:
-                        if item["nick"] in num_chans:
-                            item["num_chans"] = num_chans[item["nick"]]
-
-
-def filter_blacklisted(user, response):
-    """
-    Low level filter to take the raw OpenSearch response and remove
-    objects from it we want to keep secret.
-    Does not return, the object is mutated in place.
-    """
-    response["redacted"] = 0
-    response["exemption"] = None
-    if user.is_superuser:
-        response["exemption"] = True
-    # is_anonymous = isinstance(user, AnonymousUser)
-    # For every hit from ES
-    for index, item in enumerate(list(response["hits"]["hits"])):
-        # For every blacklisted type
-        for blacklisted_type in settings.OPENSEARCH_BLACKLISTED.keys():
-            # Check this field we are matching exists
-            if blacklisted_type in item["_source"].keys():
-                content = item["_source"][blacklisted_type]
-                # For every item in the blacklisted array for the type
-                for blacklisted_item in settings.OPENSEARCH_BLACKLISTED[
-                    blacklisted_type
-                ]:
-                    if blacklisted_item == str(content):
-                        # Remove the item
-                        if item in response["hits"]["hits"]:
-                            # Let the UI know something was redacted
-                            if (
-                                "exemption"
-                                not in response["hits"]["hits"][index]["_source"]
-                            ):
-                                response["redacted"] += 1
-                            # Anonymous
-                            if user.is_anonymous:
-                                # Just set it to none so the index is not off
-                                response["hits"]["hits"][index] = None
-                            else:
-                                if not user.is_superuser:
-                                    response["hits"]["hits"][index] = None
-                                else:
-                                    response["hits"]["hits"][index]["_source"][
-                                        "exemption"
-                                    ] = True
-
-    # Actually get rid of all the things we set to None
-    response["hits"]["hits"] = [hit for hit in response["hits"]["hits"] if hit]
-
-
-def run_main_query(client, user, query, custom_query=False, index=None, size=None):
-    """
-    Low level helper to run an ES query.
-    Accept a user to pass it to the filter, so we can
-    avoid filtering for superusers.
-    Accept fields and size, for the fields we want to match and the
-    number of results to return.
-    """
-    if not index:
-        index = settings.OPENSEARCH_INDEX_MAIN
-    if custom_query:
-        search_query = query
-    else:
-        search_query = construct_query(query, size)
-    try:
-        response = client.search(body=search_query, index=index)
-    except RequestError as err:
-        print("OpenSearch error", err)
-        return err
-    filter_blacklisted(user, response)
-    return response
-
-
-def query_results(request, query_params, size=None):
-    """
-    API helper to alter the OpenSearch return format into something
-    a bit better to parse.
-    Accept a HTTP request object. Run the query, and annotate the
-    results with the other data we have.
-    """
-    # is_anonymous = isinstance(request.user, AnonymousUser)
-    message = None
-    message_class = None
-    add_bool = []
-    add_top = []
-    add_top_negative = []
-    sort = None
-    if request.user.is_anonymous:
-        sizes = settings.OPENSEARCH_MAIN_SIZES_ANON
-    else:
-        sizes = settings.OPENSEARCH_MAIN_SIZES
-    if not size:
-        if "size" in query_params:
-            size = query_params["size"]
-            if size not in sizes:
-                message = "Size is not permitted"
-                message_class = "danger"
-                return {"message": message, "class": message_class}
-    if "source" in query_params:
-        source = query_params["source"]
-        if source not in settings.OPENSEARCH_MAIN_SOURCES:
-            message = "Invalid source"
-            message_class = "danger"
-            return {"message": message, "class": message_class}
-        if source != "all":
-            add_bool.append({"src": source})
-
-    if set({"from_date", "to_date", "from_time", "to_time"}).issubset(
-        query_params.keys()
-    ):
-        from_ts = f"{query_params['from_date']}T{query_params['from_time']}Z"
-        to_ts = f"{query_params['to_date']}T{query_params['to_time']}Z"
-        print("from ts", from_ts)
-        print("to_ts", to_ts)
-        range_query = {
-            "range": {
-                "ts": {
-                    "gt": from_ts,
-                    "lt": to_ts,
-                }
-            }
-        }
-        add_top.append(range_query)
-    if "sorting" in query_params:
-        sorting = query_params["sorting"]
-        if sorting not in ("asc", "desc", "none"):
-            message = "Invalid sort"
-            message_class = "danger"
-            return {"message": message, "class": message_class}
-        if sorting in ("asc", "desc"):
-            sort = [
-                {
-                    "ts": {
-                        "order": sorting,
-                    }
-                }
-            ]
-
-    if "check_sentiment" in query_params:
-        if "sentiment_method" not in query_params:
-            message = "No sentiment method"
-            message_class = "danger"
-            return {"message": message, "class": message_class}
-        if "sentiment" in query_params:
-            sentiment = query_params["sentiment"]
-            try:
-                sentiment = float(sentiment)
-            except ValueError:
-                message = "Sentiment is not a float"
-                message_class = "danger"
-                return {"message": message, "class": message_class}
-        sentiment_method = query_params["sentiment_method"]
-        range_query_compare = {"range": {"sentiment": {}}}
-        range_query_precise = {
-            "match": {
-                "sentiment": None,
-            }
-        }
-        if sentiment_method == "below":
-            range_query_compare["range"]["sentiment"]["lt"] = sentiment
-            add_top.append(range_query_compare)
-        elif sentiment_method == "above":
-            range_query_compare["range"]["sentiment"]["gt"] = sentiment
-            add_top.append(range_query_compare)
-        elif sentiment_method == "exact":
-            range_query_precise["match"]["sentiment"] = sentiment
-            add_top.append(range_query_precise)
-        elif sentiment_method == "nonzero":
-            range_query_precise["match"]["sentiment"] = 0
-            add_top_negative.append(range_query_precise)
-
-    if "query" in query_params:
-        query = query_params["query"]
-        search_query = construct_query(query, size)
-        if add_bool:
-            for item in add_bool:
-                search_query["query"]["bool"]["must"].append({"match": item})
-        if add_top:
-            for item in add_top:
-                search_query["query"]["bool"]["must"].append(item)
-        if add_top_negative:
-            for item in add_top_negative:
-                if "must_not" in search_query["query"]["bool"]:
-                    search_query["query"]["bool"]["must_not"].append(item)
-                else:
-                    search_query["query"]["bool"]["must_not"] = [item]
-        if sort:
-            search_query["sort"] = sort
-        results = run_main_query(
-            client,
-            request.user,  # passed through run_main_query to filter_blacklisted
-            search_query,
-            custom_query=True,
-            size=size,
-        )
-        if not results:
-            return False
-        if isinstance(results, RequestError):
-            message = results.info["error"]["root_cause"][0]["reason"]
-            message_class = "danger"
-            return {"message": message, "class": message_class}
-        if len(results["hits"]["hits"]) == 0:
-            message = "No results."
-            message_class = "danger"
-            return {"message": message, "class": message_class}
-
-        results_parsed = []
-        if "hits" in results.keys():
-            if "hits" in results["hits"]:
-                for item in results["hits"]["hits"]:
-                    element = item["_source"]
-                    element["id"] = item["_id"]
-
-                    # Split the timestamp into date and time
-                    if "ts" not in element:
-                        if "time" in element:  # will fix data later
-                            ts = element["time"]
-                            del element["time"]
-                            element["ts"] = ts
-                    if "ts" in element:
-                        ts = element["ts"]
-                        ts_spl = ts.split("T")
-                        date = ts_spl[0]
-                        time = ts_spl[1]
-                        element["date"] = date
-                        element["time"] = time
-                    results_parsed.append(element)
-        annotate_results(results_parsed)
-
-        context = {
-            "query": query,
-            "results": results_parsed,
-            "card": results["hits"]["total"]["value"],
-            "took": results["took"],
-            "redacted": results["redacted"],
-            "exemption": results["exemption"],
-        }
-        return context
-
-
-def query_single_result(request):
-    context = query_results(request, 1)
-    dedup_set = {item["nick"] for item in context["results"]}
-    if dedup_set:
-        context["item"] = context["results"][0]
-
-    return (1, context)
-
-
-def construct_query(query, size):
-    """
-    Accept some query parameters and construct an OpenSearch query.
-    """
-    if not size:
-        size = 5
-    query = {
-        "size": size,
-        "query": {
-            "bool": {
-                "must": [
-                    {
-                        "query_string": {
-                            "query": query,
-                            # "fields": fields,
-                            # "default_field": "msg",
-                            # "type": "best_fields",
-                            "fuzziness": "AUTO",
-                            "fuzzy_transpositions": True,
-                            "fuzzy_max_expansions": 50,
-                            "fuzzy_prefix_length": 0,
-                            # "minimum_should_match": 1,
-                            "default_operator": "or",
-                            "analyzer": "standard",
-                            "lenient": True,
-                            "boost": 1,
-                            "allow_leading_wildcard": True,
-                            # "enable_position_increments": False,
-                            "phrase_slop": 3,
-                            # "max_determinized_states": 10000,
-                            "quote_field_suffix": "",
-                            "quote_analyzer": "standard",
-                            "analyze_wildcard": False,
-                            "auto_generate_synonyms_phrase_query": True,
-                        }
-                    }
-                ]
-            }
-        },
-    }
-    return query

core/migrations/0007_perms.py

@@ -0,0 +1,22 @@
+# Generated by Django 4.0.6 on 2022-08-16 18:04
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0006_contentblock_page'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Perms',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+            ],
+            options={
+                'permissions': (('bypass_hashing', 'Can bypass field hashing'), ('bypass_blacklist', 'Can bypass the blacklist'), ('bypass_encryption', 'Can bypass field encryption'), ('post_irc', 'Can post to IRC'), ('post_discord', 'Can post to Discord')),
+            },
+        ),
+    ]

core/migrations/0008_alter_perms_options.py

@@ -0,0 +1,17 @@
+# Generated by Django 4.0.6 on 2022-08-27 11:50
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0007_perms'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='perms',
+            options={'permissions': (('bypass_hashing', 'Can bypass field hashing'), ('bypass_blacklist', 'Can bypass the blacklist'), ('bypass_encryption', 'Can bypass field encryption'), ('bypass_obfuscation', 'Can bypass field obfuscation'), ('bypass_delay', 'Can bypass data delay'), ('post_irc', 'Can post to IRC'), ('post_discord', 'Can post to Discord'), ('query_search', 'Can search with query strings'), ('use_insights', 'Can use the Insights page'), ('index_int', 'Can use the internal index'), ('index_meta', 'Can use the meta index'))},
+        ),
+    ]

core/migrations/0009_alter_perms_options.py

@@ -0,0 +1,17 @@
+# Generated by Django 4.0.6 on 2022-08-27 12:05
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0008_alter_perms_options'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='perms',
+            options={'permissions': (('bypass_hashing', 'Can bypass field hashing'), ('bypass_blacklist', 'Can bypass the blacklist'), ('bypass_encryption', 'Can bypass field encryption'), ('bypass_obfuscation', 'Can bypass field obfuscation'), ('bypass_delay', 'Can bypass data delay'), ('bypass_randomisation', 'Can bypass data randomisation'), ('post_irc', 'Can post to IRC'), ('post_discord', 'Can post to Discord'), ('query_search', 'Can search with query strings'), ('use_insights', 'Can use the Insights page'), ('index_int', 'Can use the internal index'), ('index_meta', 'Can use the meta index'))},
+        ),
+    ]

core/migrations/0010_alter_perms_options.py

@@ -0,0 +1,17 @@
+# Generated by Django 4.1 on 2022-09-01 11:07
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0009_alter_perms_options'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='perms',
+            options={'permissions': (('bypass_hashing', 'Can bypass field hashing'), ('bypass_blacklist', 'Can bypass the blacklist'), ('bypass_encryption', 'Can bypass field encryption'), ('bypass_obfuscation', 'Can bypass field obfuscation'), ('bypass_delay', 'Can bypass data delay'), ('bypass_randomisation', 'Can bypass data randomisation'), ('post_irc', 'Can post to IRC'), ('post_discord', 'Can post to Discord'), ('query_search', 'Can search with query strings'), ('use_insights', 'Can use the Insights page'), ('index_int', 'Can use the internal index'), ('index_meta', 'Can use the meta index'), ('restricted_sources', 'Can access restricted sources'))},
+        ),
+    ]

core/models.py

@@ -102,3 +102,22 @@ class ContentBlock(models.Model):
             self.image3 = None
 
         super().save(*args, **kwargs)
+
+
+class Perms(models.Model):
+    class Meta:
+        permissions = (
+            ("bypass_hashing", "Can bypass field hashing"),  #
+            ("bypass_blacklist", "Can bypass the blacklist"),  #
+            ("bypass_encryption", "Can bypass field encryption"),  #
+            ("bypass_obfuscation", "Can bypass field obfuscation"),  #
+            ("bypass_delay", "Can bypass data delay"),  #
+            ("bypass_randomisation", "Can bypass data randomisation"),  #
+            ("post_irc", "Can post to IRC"),
+            ("post_discord", "Can post to Discord"),
+            ("query_search", "Can search with query strings"),  #
+            ("use_insights", "Can use the Insights page"),
+            ("index_int", "Can use the internal index"),
+            ("index_meta", "Can use the meta index"),
+            ("restricted_sources", "Can access restricted sources"),
+        )

core/static/chart.js

@@ -4,40 +4,44 @@ function loadJson(selector) {
 var jsonData = loadJson('#jsonData');
 var full_data = jsonData.map((item) => item);
 
-var ctx = document.getElementById('volume').getContext("2d");
+var ctx = document.getElementById('sentiment-chart').getContext("2d");
 new Chart(ctx, {
-  type: 'line',
-  data: {
-    datasets: [
-      {
-        label: "Sentiment",
-        fill: false,
-        backgroundColor: 'black',
-        borderColor: 'lightblue',
-        tension: 0.3,
-        data: full_data,
-        spanGaps: true,
-      }
-    ],
-  },
-  options: {
-    responsive: true,
-    maintainAspectRatio: false,
-    parsing: {
-      xAxisKey: 'date',
-      yAxisKey: 'value',
+    type: 'line',
+    data: {
+        datasets: [
+            {
+                label: "Sentiment",
+                fill: false,
+                backgroundColor: 'black',
+                borderColor: 'lightblue',
+                tension: 0.3,
+                data: full_data,
+                spanGaps: true,
+            }
+        ],
     },
-    plugins: {
-      tooltip: {
-        callbacks: {
-          beforeFooter: function(context) {
-            return "Nick: " + full_data[context[0].dataIndex].nick;
-          },
-          footer: function(context) {
-            return "Msg: " + full_data[context[0].dataIndex].text;
-          }
+    options: {
+        responsive: true,
+        maintainAspectRatio: false,
+        parsing: {
+            xAxisKey: 'date',
+            yAxisKey: 'value',
+        },
+        plugins: {
+            tooltip: {
+                callbacks: {
+                    footer: function(context) {
+                        var foot = "Text: " + full_data[context[0].dataIndex].text + "\n";
+                        foot += "Nick: " + full_data[context[0].dataIndex].nick + "\n";
+                        foot += "Channel: " + full_data[context[0].dataIndex].channel + "\n";
+                        foot += "Net: " + full_data[context[0].dataIndex].net;
+                        return foot;
+                    }
+                }
+            },
+            legend: {
+                display: false,
+            },
         }
-      }
     }
-  }
 });

core/static/js/column-shifter.js

@@ -0,0 +1,246 @@
+// Author: Grzegorz Tężycki
+
+$(document).ready(function(){
+
+    // In web storage is saved structure like that:
+    // localStorage['django_tables2_column_shifter'] = {
+    //     'table_class_container1' : {
+    //         'id' : 'on',
+    //         'col1' : 'off',
+    //         'col2' : 'on',
+    //         'col3' : 'on',
+    //     },
+    //     'table_class_container2' : {
+    //         'id' : 'on',
+    //         'col1' : 'on'
+    //     },
+    // }
+
+    // main name for key in web storage
+    var COLUMN_SHIFTER_STORAGE_ACCESOR = "django_tables2_column_shifter";
+
+    // Return storage structure for shifter
+    // If structure does'n exist in web storage
+    // will be return empty object
+    var get_column_shifter_storage = function(){
+        var storage = localStorage.getItem(COLUMN_SHIFTER_STORAGE_ACCESOR);
+        if (storage === null) {
+            storage = {
+                "drilldown-table": {
+                    "date": "off",
+                    "time": "off",
+                    "id": "off",
+                    "host": "off",
+                    "ident": "off",
+                    "channel": "off",
+                    "net": "off",
+                    "num": "off",
+                    "channel_nsfw": "off",
+                    "channel_category": "off",
+                    "channel_category_id": "off",
+                    "channel_category_nsfw": "off",
+                    "channel_id": "off",
+                    "guild_member_count": "off",
+                    "bot": "off",
+                    "msg_id": "off",
+                    "net_id": "off",
+                    "user_id": "off",
+                    "nick_id": "off",
+                    "status": "off",
+                    "num_users": "off",
+                    "num_chans": "off",
+                    "exemption": "off",
+                    // "version_sentiment": "off",
+                    "sentiment": "off",
+                    "num": "off",
+                    "online": "off",
+                    "mtype": "off",
+                    "realname": "off",
+                    "server": "off",
+                    "mtype": "off",
+                    "hidden": "off",
+                    "filename": "off",
+                    "file_md5": "off",
+                    "file_ext": "off",
+                    "file_size": "off",
+                },
+            };
+        } else {
+            storage = JSON.parse(storage);
+        }
+        return storage;
+    };
+
+    // Save structure in web storage
+    var set_column_shifter_storage = function(storage){
+        var json_storage = JSON.stringify(storage)
+        localStorage.setItem(COLUMN_SHIFTER_STORAGE_ACCESOR, json_storage);
+    };
+
+    // Remember state for single button
+    var save_btn_state = function($btn){
+
+        // Take css class for container with table
+        var table_class_container = $btn.data("table-class-container");
+        // Take html object with table
+        var $table_class_container = $("#" + table_class_container);
+        // Take single button statne ("on" / "off")
+        var state = $btn.data("state");
+        // td-class is a real column name in table
+        var td_class = $btn.data("td-class");
+        var storage = get_column_shifter_storage();
+        // Table id
+        var id = $table_class_container.attr("id");
+
+        // Checking if the ID is already in storage
+        if (id in storage) {
+            data = storage[id]
+        } else {
+            data = {}
+            storage[id] = data;
+        }
+
+        // Save state for table column in storage
+        data[td_class] = state;
+        set_column_shifter_storage(storage);
+    };
+
+    // Load states for buttons from storage for single tabel
+    var load_states = function($table_class_container) {
+        var storage = get_column_shifter_storage();
+        // Table id
+        var id = $table_class_container.attr("id");
+        var data = {};
+
+        // Checking if the ID is already in storage
+        if (id in storage) {
+            data = storage[id]
+
+            // For each shifter button set state
+            $table_class_container.find(".btn-shift-column").each(function(){
+                var $btn = $(this);
+                var td_class = $btn.data("td-class");
+
+                // If name of column is in store then get state
+                // and set state
+                if (td_class in data) {
+                    var state = data[td_class]
+                    set_btn_state($btn, state);
+                }
+            });
+        }
+    };
+
+    // Show table content and hide spiner
+    var show_table_content = function($table_class_container){
+        $table_class_container.find("#loader").hide();
+        $table_class_container.find("#table-container").show();
+    };
+
+    // Load buttons states for all button in page
+    var load_state_for_all_containters = function(){
+        $(".column-shifter-container").each(function(){
+            $table_class_container = $(this);
+
+            // Load states for all buttons in single container
+            load_states($table_class_container);
+
+            // When states was loaded then table must be show and
+            // loader (spiner) must be hide
+            show_table_content($table_class_container);
+        });
+    };
+
+    // change visibility column for single button
+    // if button has state "on" then show column
+    // else then column will be hide
+    shift_column = function( $btn ){
+        // button state
+        var state = $btn.data("state");
+
+        // td-class is a real column name in table
+        var td_class = $btn.data("td-class");
+        var table_class_container = $btn.data("table-class-container");
+        var $table_class_container = $("#" + table_class_container);
+        var $table = $table_class_container.find("table");
+        var $cels = $table.find("." + td_class);
+
+        if ( state === "on" ) {
+            $cels.show();
+        } else {
+            $cels.hide();
+        }
+    };
+
+    // Shift visibility for all columns
+    shift_columns = function(){
+        var cols = $(".btn-shift-column");
+        var i, len = cols.length;
+        for (i=0; i < len; i++) {
+            shift_column($(cols[i]));
+        }
+    };
+
+    // Set icon imgae visibility for button state
+    var set_icon_for_state = function( $btn, state ) {
+        if (state === "on") {
+            $btn.find("span.uncheck").hide();
+            $btn.find("span.check").show();
+        } else {
+            $btn.find("span.check").hide();
+            $btn.find("span.uncheck").show();
+        }
+    };
+
+    // Set state for single button
+    var set_btn_state = function($btn, state){
+        $btn.data('state', state);
+        set_icon_for_state($btn, state);
+    }
+
+    // Change state for single button
+    var change_btn_state = function($btn){
+        var state = $btn.data("state");
+
+        if (state === "on") {
+            state = "off"
+        } else {
+            state = "on"
+        }
+        set_btn_state($btn, state);
+    };
+
+    // Run show/hide when click on button
+    $(".btn-shift-column").on("click", function(event){
+        var $btn = $(this);
+        event.stopPropagation();
+        change_btn_state($btn);
+        shift_column($btn);
+        save_btn_state($btn);
+    });
+
+    // Load saved states for all tables
+    load_state_for_all_containters();
+
+    // show or hide columns based on data from web storage
+    shift_columns();
+
+    // Add API method for retrieving non-visible cols for table
+    // Pass the 0-based index of the table or leave the parameter
+    // empty to return the hidden cols for the 1st table found
+    $.django_tables2_column_shifter_hidden = function(idx) {
+        if(idx==undefined) {
+            idx = 0;
+        }
+        return $('#table-container').eq(idx).find('.btn-shift-column').filter(function(z) {
+            return $(this).data('state')=='off'
+        }).map(function(z) {
+            return $(this).data('td-class')
+        }).toArray();
+    }
+    const event = new Event('restore-scroll');
+    document.dispatchEvent(event);
+    const event2 = new Event('load-widget-results');
+    document.dispatchEvent(event2);
+
+});

core/static/modal.js

@@ -1,37 +1,44 @@
-var modal = document.querySelector('.modal');  // assuming you have only 1
+// var modal = document.querySelector('.modal');  // assuming you have only 1
+var modal = document.getElementById("modal");
 var html = document.querySelector('html');
 
+var disableModal = function() {
+    modal.classList.remove('is-active');
+    html.classList.remove('is-clipped');
+    var modal_refresh = document.getElementsByClassName("modal-refresh");
+    for(var i = 0; i < modal_refresh.length; i++) {
+        modal_refresh[i].remove();
+    }
+}
+
 var elements = document.querySelectorAll('.modal-background');
 for(var i = 0; i < elements.length; i++) {
     elements[i].addEventListener('click', function(e) {
-      // elements[i].preventDefault();
-      modal.classList.remove('is-active');
-      html.classList.remove('is-clipped');
+        // elements[i].preventDefault();
+        disableModal();
     });
 }
 
 var elements = document.querySelectorAll('.modal-close');
 for(var i = 0; i < elements.length; i++) {
     elements[i].addEventListener('click', function(e) {
-      // elements[i].preventDefault();
-      modal.classList.remove('is-active');
-      html.classList.remove('is-clipped');
+        // elements[i].preventDefault();
+        disableModal();
     });
 }
 
 function activateButtons() {
-  var elements = document.querySelectorAll('.modal-close-button');
-  for(var i = 0; i < elements.length; i++) {
-      elements[i].addEventListener('click', function(e) {
-        // elements[i].preventDefault();
-        modal.classList.remove('is-active');
-        html.classList.remove('is-clipped');
-      });
-  }
+    var elements = document.querySelectorAll('.modal-close-button');
+    for(var i = 0; i < elements.length; i++) {
+        elements[i].addEventListener('click', function(e) {
+            // elements[i].preventDefault();
+            disableModal();
+        });
+    }
 }
 activateButtons();
 // modal.querySelector('.modal-close-button').addEventListener('click', function(e) {
 //   e.preventDefault();
 //   modal.classList.remove('is-active');
 //   html.classList.remove('is-clipped');
-// });
+// });

core/static/style.css

@@ -1,21 +1,21 @@
 .register a {
-  color: #700000 !important;
-  margin-left: 8px;
+    color: #700000 !important;
+    margin-left: 8px;
 }
 .title {
-  color: white;
+    color: white;
 }
 body{
-  background-color: #252525;
+    background-color: #252525;
 }
 .register p {
-  font-size: 12px;
-  padding-top: 10px;
+    font-size: 12px;
+    padding-top: 10px;
 }
 .logo {
-  display: block;
-  margin-left: auto;
-  margin-right: auto;
+    display: block;
+    margin-left: auto;
+    margin-right: auto;
 }
 
 .vertical-offset-100{
@@ -23,61 +23,61 @@ body{
 }
 
 .product-container {
-  background: #ffffff;
+    background: #ffffff;
 
-  justify-content: space-between;
+    justify-content: space-between;
 }
 .product {
-  display: flex;
+    display: flex;
 }
 .description {
-  display: flex;
-  flex-direction: column;
-  justify-content: center;
+    display: flex;
+    flex-direction: column;
+    justify-content: center;
 }
 
 img {
-  border-radius: 6px;
-  margin: 10px;
-  width: 54px;
-  height: 57px;
+    border-radius: 6px;
+    margin: 10px;
+    width: 54px;
+    height: 57px;
 }
 h3,
 h5 {
-  font-style: normal;
-  font-weight: 500;
-  font-size: 14px;
-  line-height: 20px;
-  letter-spacing: -0.154px;
-  color: #242d60;
-  margin: 0;
+    font-style: normal;
+    font-weight: 500;
+    font-size: 14px;
+    line-height: 20px;
+    letter-spacing: -0.154px;
+    color: #242d60;
+    margin: 0;
 }
 h5 {
-  opacity: 0.5;
+    opacity: 0.5;
 }
 #checkout-button, #setup-button, #button {
-  height: 36px;
-  background: #556cd6;
-  color: white;
-  width: 100%;
-  font-size: 14px;
-  border: 0;
-  font-weight: 500;
-  cursor: pointer;
-  letter-spacing: 0.6;
-  border-radius: 0 0 6px 6px;
-  transition: all 0.2s ease;
-  box-shadow: 0px 4px 5.5px 0px rgba(0, 0, 0, 0.07);
+    height: 36px;
+    background: #556cd6;
+    color: white;
+    width: 100%;
+    font-size: 14px;
+    border: 0;
+    font-weight: 500;
+    cursor: pointer;
+    letter-spacing: 0.6;
+    border-radius: 0 0 6px 6px;
+    transition: all 0.2s ease;
+    box-shadow: 0px 4px 5.5px 0px rgba(0, 0, 0, 0.07);
 }
 #checkout-button:hover {
-  opacity: 0.8;
+    opacity: 0.8;
 }
 
 
 .subtitle {
-  color: #dddddd;
+    color: #dddddd;
 }
 
 .search-bar {
-  margin-bottom: 1rem;
+    margin-bottom: 1rem;
 }

core/static/tabs.css

@@ -1,7 +0,0 @@
-#tab-content div {
-    display: none;
-}
-
-#tab-content div.is-active {
-    display: block;
-}

core/static/tabs.js

@@ -1,35 +1,36 @@
-    // tabbed browsing for the modal
-    function initTabs() {
-        TABS.forEach((tab) => {
-          tab.addEventListener('click', (e) => {
+function initTabs(unique) {
+    var tabs_selector = '#tabs-'+unique+' li';
+    var TABS = [...document.querySelectorAll(tabs_selector)];
+    var CONTENT = [...document.querySelectorAll('#tab-content-'+unique+' div')];
+    var ACTIVE_CLASS = 'is-active';
+    TABS.forEach((tab) => {
+        tab.addEventListener('click', (e) => {
             let selected = tab.getAttribute('data-tab');
-            updateActiveTab(tab);
-            updateActiveContent(selected);
-          })
+            updateActiveTab(TABS, ACTIVE_CLASS, tab);
+            updateActiveContent(CONTENT, ACTIVE_CLASS, selected);
         })
-      }
-    
-      function updateActiveTab(selected) {
-        TABS.forEach((tab) => {
-          if (tab && tab.classList.contains(ACTIVE_CLASS)) {
+    })
+}
+
+function updateActiveTab(TABS, ACTIVE_CLASS, selected) {
+    TABS.forEach((tab) => {
+        if (tab && tab.classList.contains(ACTIVE_CLASS)) {
             tab.classList.remove(ACTIVE_CLASS);
-          }
-        });
-        selected.classList.add(ACTIVE_CLASS);
-      }
-    
-      function updateActiveContent(selected) {
-        CONTENT.forEach((item) => {
-          if (item && item.classList.contains(ACTIVE_CLASS)) {
+        }
+    });
+    selected.classList.add(ACTIVE_CLASS);
+}
+
+function updateActiveContent(CONTENT, ACTIVE_CLASS, selected) {
+    CONTENT.forEach((item) => {
+        if (item && item.classList.contains(ACTIVE_CLASS)) {
             item.classList.remove(ACTIVE_CLASS);
-          }
-          let data = item.getAttribute('data-content');
-          if (data === selected) {
+        }
+        let data = item.getAttribute('data-content');
+        if (data === selected) {
             item.classList.add(ACTIVE_CLASS);
-          }
-        });
-      }
-var TABS = [...document.querySelectorAll('#tabs li')];
-var CONTENT = [...document.querySelectorAll('#tab-content div')];
-var ACTIVE_CLASS = 'is-active';
-initTabs();
+        }
+    });
+}
+
+// initTabs();

core/templates/base.html

@@ -15,43 +15,61 @@
     <link rel="stylesheet" href="{% static 'css/bulma-slider.min.css' %}">
     <link rel="stylesheet" href="{% static 'css/bulma-calendar.min.css' %}">
     <link rel="stylesheet" href="{% static 'css/bulma-tagsinput.min.css' %}">
+    <link rel="stylesheet" href="{% static 'css/bulma-switch.min.css' %}">
+    <link rel="stylesheet" href="{% static 'css/gridstack.min.css' %}">
     <script src="{% static 'js/bulma-calendar.min.js' %}" integrity="sha384-DThNif0xGXbopX7+PE+UabkuClfI/zELNhaVqoGLutaWB76dyMw0vIQBGmUxSfVQ" crossorigin="anonymous"></script>
     <script src="{% static 'js/bulma-slider.min.js' %}" integrity="sha384-wbyps8iLG8QzJE02viYc/27BtT5HSa11+b5V7QPR1/huVuA8f4LRTNGc82qAIeIZ" crossorigin="anonymous"></script>
-    <script defer src="{% static 'js/htmx.min.js' %}" integrity="sha384-cZuAZ+ZbwkNRnrKi05G/fjBX+azI9DNOkNYysZ0I/X5ZFgsmMiBXgDZof30F5ofc" crossorigin="anonymous"></script>
+    <script src="{% static 'js/htmx.min.js' %}" integrity="sha384-cZuAZ+ZbwkNRnrKi05G/fjBX+azI9DNOkNYysZ0I/X5ZFgsmMiBXgDZof30F5ofc" crossorigin="anonymous"></script>
     <script defer src="{% static 'js/remove-me.js' %}" integrity="sha384-6fHcFNoQ8QEI3ZDgw9Z/A6Brk64gF7AnFbLgdrumo8/kBbsKQ/wo7wPegj5WkzuG" crossorigin="anonymous"></script>
     <script defer src="{% static 'js/hyperscript.min.js' %}" integrity="sha384-6GYN8BDHOJkkru6zcpGOUa//1mn+5iZ/MyT6mq34WFIpuOeLF52kSi721q0SsYF9" crossorigin="anonymous"></script>
-    <script src="{% static 'js/bulma-tagsinput.min.js' %}" integrity="sha384-GmnKCsPJIPPZbNVXpkGRmKdxOa0PQLnOM/hQLIHvMRERySuyvFqKGc76iHTGUY+d" crossorigin="anonymous"></script>
+    <script src="{% static 'js/bulma-tagsinput.min.js' %}"></script>
+    <script src="{% static 'js/jquery.min.js' %}"></script>
+    <script src="{% static 'js/gridstack-all.js' %}"></script>
+    <script defer src="{% static 'js/magnet.min.js' %}"></script>
+    <script>
+      document.addEventListener("restore-scroll", function(event) {
+        var scrollpos = localStorage.getItem('scrollpos');
+        if (scrollpos) {
+          window.scrollTo(0, scrollpos)
+        };
+      });
+
+      document.addEventListener("htmx:beforeSwap", function(event) {
+        localStorage.setItem('scrollpos', window.scrollY);
+
+      });
+    </script>
     <script>
       document.addEventListener('DOMContentLoaded', () => {
 
         // Get all "navbar-burger" elements
         const $navbarBurgers = Array.prototype.slice.call(document.querySelectorAll('.navbar-burger'), 0);
-      
+
         // Add a click event on each of them
         $navbarBurgers.forEach( el => {
           el.addEventListener('click', () => {
-      
+
             // Get the target from the "data-target" attribute
             const target = el.dataset.target;
             const $target = document.getElementById(target);
-      
+
             // Toggle the "is-active" class on both the "navbar-burger" and the "navbar-menu"
             el.classList.toggle('is-active');
             $target.classList.toggle('is-active');
-      
+
           });
         });
-      
+
       });
     </script>
     <style>
       .icon { border-bottom: 0px !important;}
       .wrap {
-        white-space: pre-wrap;      /* CSS3 */
-        white-space: -moz-pre-wrap; /* Firefox */
-        white-space: -pre-wrap;     /* Opera <7 */
-        white-space: -o-pre-wrap;   /* Opera 7 */
-        word-wrap: break-word;      /* IE */
+        /* white-space: pre-wrap;
+        white-space: -moz-pre-wrap;
+        white-space: -pre-wrap;
+        white-space: -o-pre-wrap; */
+        word-wrap: break-word;
       }
       .nowrap-parent {
         white-space: nowrap;
@@ -67,13 +85,133 @@
         transition: opacity 500ms ease-in;
       }
       .htmx-request .htmx-indicator{
-          opacity:1
+        opacity:1
       }
       .htmx-request.htmx-indicator{
-          opacity:1
+        opacity:1
+      }
+      .dropdown-content {
+        height: 20em;
+        overflow: auto;
+      }
+      table.relays-table tr:nth-of-type(2n) td {
+        border-bottom: 3px solid grey;
+      }
+
+      .tooltiptext {
+        visibility: hidden;
+        background-color: black;
+        color: #fff;
+        text-align: center;
+        padding: 5px 0;
+        border-radius: 6px;
+        position: absolute;
+        z-index: 1;
+      }
+
+      .rounded-tooltip:hover .tooltiptext {
+        visibility: visible;
+      }
+
+      #sentiment-container {
+        position: fixed;
+        top: 0;
+        right: 0;
+        bottom: 0;
+        left: 0;
+        height: 100vh;
+        width: 100vw;
+        z-index: -2;
+      }
+
+      .table {
+        background: transparent !important;
+      }
+
+      tr {
+        transition: all 0.2s ease-in-out;
+      }
+
+      tr:hover {
+        cursor:pointer;
+        background-color:rgba(221, 224, 255, 0.3) !important;
+      }
+
+      a.panel-block {
+        transition: all 0.2s ease-in-out;
+      }
+
+      a.panel-block:hover {
+        cursor:pointer;
+        background-color:rgba(221, 224, 255, 0.3) !important;
+      }
+
+      .panel, .box, .modal {
+        background-color:rgba(250, 250, 250, 0.5) !important;
+      }
+      .modal, .modal.box{
+        background-color:rgba(210, 210, 210, 0.9) !important;
+      }
+      .modal-background{
+        background-color:rgba(255, 255, 255, 0.3) !important;
+      }
+
+      .has-background-grey-lighter{
+        background-color:rgba(219, 219, 219, 0.5) !important;
+      }
+      .navbar {
+        background-color:rgba(0, 0, 0, 0.03) !important;
+      }
+
+      .grid-stack-item-content {
+        display: flex !important;
+        flex-direction: column !important;
+        overflow-x: hidden !important;
+        overflow-y: hidden !important;
+      }
+
+      .panel {
+        display: flex !important;
+        flex-direction: column !important;
+        overflow: hidden;
+      }
+
+      .panel-block {
+        overflow-y:auto;
+        overflow-x:auto;
+        min-height: 90%;
+        display: block;
+      }
+
+      .floating-window {
+        /* background-color:rgba(210, 210, 210, 0.6) !important; */
+        display: flex !important;
+        flex-direction: column !important;
+        overflow-x: hidden !important;
+        overflow-y: hidden !important;
+        max-height: 300px;
+        z-index: 9000;
+        position: absolute;
+        top: 50px;
+        left: 50px;
+      }
+
+      .floating-window .panel {
+        background-color:rgba(250, 250, 250, 0.8) !important;
+      }
+
+      .float-right {
+        float: right;
+        padding-right: 5px;
+        padding-left: 5px;
+      }
+      .grid-stack-item:hover .ui-resizable-handle {
+        display: block !important;
+      }
+      .ui-resizable-handle {
+        z-index: 39 !important;
       }
 
-      
     </style>
   </head>
   <body>
@@ -83,22 +221,19 @@
         <a class="navbar-item" href="{% url 'home' %}">
           <img src="{% static 'logo.svg' %}" width="112" height="28" alt="logo">
         </a>
-    
+
         <a role="button" class="navbar-burger" aria-label="menu" aria-expanded="false" data-target="bar">
           <span aria-hidden="true"></span>
           <span aria-hidden="true"></span>
           <span aria-hidden="true"></span>
         </a>
       </div>
-    
+
       <div id="bar" class="navbar-menu">
         <div class="navbar-start">
           <a class="navbar-item" href="{% url 'home' %}">
             Search
           </a>
-          <a class="navbar-item" href="{% url 'about' %}">
-            About
-          </a>
           {% if user.is_authenticated %}
             <a class="navbar-item" href="{% url 'billing' %}">
               Billing
@@ -109,7 +244,7 @@
               <a class="navbar-link">
                 Threshold
               </a>
-      
+
               <div class="navbar-dropdown">
                 <a class="navbar-item" href="{% url 'threshold_irc_overview' %}">
                   IRC
@@ -121,18 +256,16 @@
             </div>
           {% endif %}
 
-          {% if user.is_authenticated %}
-            {% if user|has_plan:'drilldown' %}
-              <a class="navbar-item" href="{% url 'insights' %}">
-                Insights
-              </a>
-            {% endif %}
+          {% if perms.core.use_insights %}
+            <a class="navbar-item" href="{# url 'insights' #}">
+              Insights
+            </a>
           {% endif %}
           <a class="navbar-item add-button">
             Install
           </a>
         </div>
-    
+
         <div class="navbar-end">
           <div class="navbar-item">
             <div class="buttons">
@@ -149,7 +282,6 @@
                 <a class="button is-dark" href="{% url 'logout' %}">Logout</a>
               {% endif %}
 
-
             </div>
           </div>
         </div>
@@ -174,16 +306,18 @@
           deferredPrompt.prompt();
           // Wait for the user to respond to the prompt
           deferredPrompt.userChoice.then((choiceResult) => {
-              if (choiceResult.outcome === 'accepted') {
-                console.log('User accepted the A2HS prompt');
-              } else {
-                console.log('User dismissed the A2HS prompt');
-              }
-              deferredPrompt = null;
-            });
+            if (choiceResult.outcome === 'accepted') {
+              console.log('User accepted the A2HS prompt');
+            } else {
+              console.log('User dismissed the A2HS prompt');
+            }
+            deferredPrompt = null;
+          });
         });
       });
     </script>
+    {% block outer_content %}
+    {% endblock %}
     <section class="section">
       <div class="container">
         {% block content %}

core/templates/billing.html

@@ -31,6 +31,16 @@
       Subscription management
     </a>
   </article>
-  {% include "partials/product-list.html" %}
+  <div class="box">
+    <h1 class="subtitle">
+      This product is currently free. You may cancel any plans above.
+    </h1>
+  </div>
+  <div class="box">
+    <h1 class="subtitle">
+      You cannot pay for access to the raw data. It is hashed to preserve privacy.
+    </h1>
+  </div>
+  {# {% include "partials/product-list.html" %} #}
 {% endblock %}
 

core/templates/manage/threshold/irc/network/actions.html

@@ -31,7 +31,20 @@
     </button>
     <button
       hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
-      hx-post="{% url 'threshold_irc_network_actions_list' net %}"
+      hx-get="{% url 'threshold_irc_actions_registration_net' net %}"
+      hx-trigger="click"
+      hx-target="#modals-here"
+      class="button is-info">
+      <span class="icon-text">
+        <span class="icon">
+          <i class="fa-solid fa-list"></i>
+        </span>
+        <span>Registration</span>
+      </span>
+    </button>
+    <button
+      hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
+      hx-post="{% url 'threshold_irc_network_list' net %}"
       hx-trigger="click"
       hx-target="#actions"
       hx-swap="outerHTML"
@@ -65,5 +78,4 @@
       </div>
     </div>
   </form>
-  
 </div>

core/templates/manage/threshold/irc/network/channels.html

@@ -6,21 +6,26 @@
         <table class="table is-fullwidth is-hoverable">
           <thead>
             <th>channel</th>
+            <th>num</th>
             <th>actions</th>
           </thead>
           <tbody>
-            {% for channel, info in channels.items %}
+            {% for channel in channels %}
               <tr>
                 <td>
-                  {{ channel }}
+                  {{ channel.name }}
                   <span class="tag">
-                    {{ info }}
+                    {{ channel.users }}
                   </span>
                 </td>
+                <td>
+                  {{ channel.num }}
+                </td>
                 <td>
                   <button
-                    hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
-                    hx-delete="{% url 'threshold_irc_network_channel' net channel %}"
+                    hx-headers='{"X-CSRFToken": "{{ csrf_token }}", "Content-Type": "application/json"}'
+                    hx-delete="{% url 'threshold_irc_network_channel_json' net %}"
+                    hx-vals='{"channel": "{{ channel.name }}"}'
                     hx-target="#channels"
                     hx-swap="outerHTML"
                     class="button is-danger is-small">

core/templates/manage/threshold/irc/network/modals/registration.html

@@ -0,0 +1,99 @@
+{% load index %}
+{% load static %}
+
+<script src="{% static 'modal.js' %}"></script>
+<div id="modal" class="modal is-active is-clipped">
+  <div class="modal-background"></div>
+  <div class="modal-content">
+    <div class="box">
+      {% include 'manage/threshold/partials/notify.html' %}
+      <h4 class="subtitle is-4">Registration</h4>
+      <div class="buttons">
+        <button
+          hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
+          hx-post="{% url 'threshold_irc_actions_registration_auth' %}"
+          hx-vals='{"net": "{{ net }}", "func": "recheckauth"}'
+          hx-trigger="click"
+          hx-target="#modals-here"
+          class="button is-info">
+          <span class="icon-text">
+            <span class="icon">
+              <i class="fa-solid fa-wrench"></i>
+            </span>
+            <span>Check auth</span>
+          </span>
+        </button>
+        <button
+          hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
+          hx-post="{% url 'threshold_irc_actions_registration_auth' %}"
+          hx-vals='{"net": "{{ net }}", "func": "resetauth"}'
+          hx-trigger="click"
+          hx-target="#modals-here"
+          class="button is-info">
+          <span class="icon-text">
+            <span class="icon">
+              <i class="fa-solid fa-wrench"></i>
+            </span>
+            <span>Reset auth</span>
+          </span>
+        </button>
+        <button
+          hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
+          hx-post="{% url 'threshold_irc_actions_registration_auth' %}"
+          hx-vals='{"net": "{{ net }}", "func": "register"}'
+          hx-trigger="click"
+          hx-target="#modals-here"
+          class="button is-info">
+          <span class="icon-text">
+            <span class="icon">
+              <i class="fa-solid fa-wrench"></i>
+            </span>
+            <span>Register</span>
+          </span>
+        </button>
+      </div>
+      {% if unreg is not None %}
+        <form
+          hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
+          hx-put="{% url 'threshold_irc_actions_registration_net' net %}"
+          hx-target="#actions"
+          hx-swap="outerHTML">
+          {% for network, items in unreg.items %}
+            <h4 class="title is-4">{{ network }}</h4>
+            {% if items is not False %}
+              {% for nick, num in items %}
+                <div class="field">
+                  <label class="label">{{ nick }}/{{ num }}</label>
+                  <div class="control">
+                    <input class="input" type="text" name="{{ network }}|{{ num }}" placeholder="Enter token">
+                  </div>
+                </div>
+              {% endfor %}
+            {% else %}
+              <p>Error getting information for {{ network }}.</p>
+            {% endif %}
+          {% endfor %}
+          <button
+            type="button"
+            class="button is-light modal-close-button">
+            Cancel
+          </button>
+          <button type="submit" class="button is-info modal-close-button">Submit</button>
+          {# <script>activateButtons();</script> #}
+        </form>
+      {% else %}
+        <p>No unregistered relays.</p>
+      {% endif %}
+
+    </div>
+    <button class="modal-close is-large" aria-label="close"></button>
+  </div>
+</div>
+
+
+
+
+
+
+
+

core/templates/manage/threshold/irc/network/network.html

@@ -1,11 +1,33 @@
 {% extends "base.html" %}
 {% block content %}
+  <script>
+    document.addEventListener("restore-relay-scroll", function(event) {
+      var modalContent = document.getElementsByClassName("relay_table_container")[0];
+      var maxScroll = modalContent.scrollHeight - modalContent.offsetHeight;
+      var scrollpos = localStorage.getItem('scrollpos_relays_table');
+      if (scrollpos == 'BOTTOM') {
+        modalContent.scrollTop = maxScroll;
+      } else if (scrollpos) {
+        modalContent.scrollTop = scrollpos;
+      };
+    });
 
+    document.addEventListener("htmx:beforeSwap", function(event) {
+      var modalContent = document.getElementsByClassName("relay_table_container")[0];
+      var scrollpos = modalContent.scrollTop;
+      if(modalContent.scrollTop === (modalContent.scrollHeight - modalContent.offsetHeight)) {
+        localStorage.setItem('scrollpos_relays_table', 'BOTTOM');
+      } else {
+        localStorage.setItem('scrollpos_relays_table', scrollpos);
+      }
+
+    });
+  </script>
   <div
     style="display: none;"
     hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
     hx-get="{% url 'threshold_irc_network_info' net %}"
-    hx-trigger="load, every 5s"
+    hx-trigger="load, every 60s"
     hx-target="#info"
     hx-swap="outerHTML">
   </div>
@@ -14,16 +36,17 @@
     style="display: none;"
     hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
     hx-get="{% url 'threshold_irc_network_relays' net %}"
-    hx-trigger="load, every 5s"
+    hx-trigger="load, every 60s"
     hx-target="#relays"
-    hx-swap="outerHTML">
+    {# hx-swap="innerHTML" #}
+  >
   </div>
 
   <div
     style="display: none;"
     hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
     hx-get="{% url 'threshold_irc_network_channels' net %}"
-    hx-trigger="load, every 5s"
+    hx-trigger="load, every 60s"
     hx-target="#channels"
     hx-swap="outerHTML">
   </div>
@@ -37,6 +60,16 @@
     hx-swap="outerHTML">
   </div>
 
+
+  <div
+    style="display: none;"
+    hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
+    hx-get="{% url 'threshold_irc_network_list' net %}"
+    hx-trigger="load"
+    hx-target="#stats"
+    hx-swap="outerHTML">
+  </div>
+
   <div class="columns">
     <div class="column">
       <div class="box">
@@ -44,11 +77,21 @@
         </div>
       </div>
     </div>
-    <div class="column">
+    <!-- <div class="column">
       <div class="box">
         <div id="relays">
         </div>
       </div>
+    </div> -->
+    <div class="column">
+      <div class="box">
+        <div>
+          <div class="content" style="max-height: 30em; overflow: auto;">
+            <div class="table-container relay_table_container" id="relays">
+            </div>
+          </div>
+        </div>
+      </div>
     </div>
   </div>
 
@@ -61,8 +104,8 @@
     </div>
     <div class="column">
       <div class="box">
-        <div id="alerts">
-          Alerts here
+        <div id="stats">
+          Stats here
         </div>
       </div>
     </div>
@@ -102,4 +145,6 @@
       </div>
     </div>
   </div>
+  <div id="modals-here">
+  </div>
 {% endblock %}

core/templates/manage/threshold/irc/network/relays.html

@@ -1,100 +1,221 @@
 {% load index %}
-<div id="relays">
-  {% include 'manage/threshold/partials/notify.html' %}
-  {% if relays is not None %}
-    <div class="content" style="max-height: 30em; overflow: auto;">
-      <div class="table-container">
-        <table class="table is-fullwidth is-hoverable">
-          <thead>
-            <th>id</th>
-            <th>reg</th>
-            <th>on</th>
-            <th>
-              <span class="icon">
-                <i class="fa-solid fa-hashtag"></i>
+
+<div class="table-container relay_table_container" id="relays">
+  <table class="table is-fullwidth is-hoverable relays-table">
+    <thead>
+      <th>id</th>
+      <th>
+        <span class="icon has-tooltip-bottom" data-tooltip="Registered">
+          <i class="fa-solid fa-seal"></i>
+        </span>
+      </th>
+      <th>
+        <span class="icon has-tooltip-bottom" data-tooltip="Authenticated">
+          <i class="fa-solid fa-passport"></i>
+        </span>
+      </th>
+      <th>
+        <span class="icon has-tooltip-bottom" data-tooltip="Connected">
+          <i class="fa-solid fa-cloud-question"></i>
+        </span>
+      </th>
+      <th>
+        <span class="icon has-tooltip-bottom" data-tooltip="Enabled">
+          <i class="fa-solid fa-toggle-on"></i>
+        </span>
+      </th>
+      <th>
+        <span class="icon has-tooltip-bottom" data-tooltip="Channels">
+          <i class="fa-solid fa-hashtag"></i>
+        </span>
+      </th>
+      <th>
+        <span class="icon has-tooltip-bottom" data-tooltip="Chanlimit">
+          <i class="fa-solid fa-list-ol"></i>
+        </span>
+      </th>
+      <th>nick</th>
+      <th>
+        <span class="icon has-tooltip-bottom" data-tooltip="Actions">
+          <i class="fa-solid fa-wrench"></i>
+        </span>
+      </th>
+    </thead>
+    <tbody>
+      {% for relay in relays %}
+        <tr>
+          <td>{{ relay.id }}</td>
+          <td>
+            {% if relay.registered %}
+              <span class="icon has-text-success">
+                <i class="fa-solid fa-check" aria-hidden="true"></i>
               </span>
-            </th>
-            <th>nick</th>
-            <th>
-              <span class="icon">
+            {% else %}
+              <span class="icon has-text-danger">
+                <i class="fa-solid fa-xmark" aria-hidden="true"></i>
+              </span>
+            {% endif %}
+          </td>
+          <td>
+            {% if relay.authed %}
+              <span class="icon has-text-success">
+                <i class="fa-solid fa-check" aria-hidden="true"></i>
+              </span>
+            {% else %}
+              <span class="icon has-text-danger">
+                <i class="fa-solid fa-xmark" aria-hidden="true"></i>
+              </span>
+            {% endif %}
+          </td>
+          <td>
+            {% if relay.conn %}
+              <span class="icon has-text-success">
+                <i class="fa-solid fa-check" aria-hidden="true"></i>
+              </span>
+            {% else %}
+              <span class="icon has-text-danger">
+                <i class="fa-solid fa-xmark" aria-hidden="true"></i>
+              </span>
+            {% endif %}
+          </td>
+          <td>
+            {% if relay.enabled %}
+              <span class="icon has-text-success">
+                <i class="fa-solid fa-check" aria-hidden="true"></i>
+              </span>
+            {% else %}
+              <span class="icon has-text-danger">
+                <i class="fa-solid fa-xmark" aria-hidden="true"></i>
+              </span>
+            {% endif %}
+          </td>
+          <td>
+            {{ relay.chans }}
+          </td>
+          <td>{{ relay.limit }}</td>
+          <td>
+            {{ relay.nick }}
+          </td>
+          <td>
+            <a
+              hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
+              hx-post="{% url 'modal_context' %}"
+              hx-vals='{"net": "{{ net }}",
+              "num": "{{ relay.id }}",
+              "source": "irc",
+              "channel": "*status",
+              "time": "None",
+              "date": "None",
+              "index": "internal",
+              "type": "znc",
+              "mtype": "None",
+              "nick": "*status",
+              "dedup": "on"}'
+              hx-target="#modals-here"
+              hx-trigger="click"
+              class="button is-small has-background-info has-text-white">
+              <span class="icon has-tooltip-left" data-tooltip="ZNC context">
+                <i class="fa-brands fa-unity" aria-hidden="true"></i>
+              </span>
+            </a>
+          </td>
+        </tr>
+        <tr>
+          <td>
+            <a
+              hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
+              hx-delete="{% url 'threshold_irc_network_relay_del' relay|index:'net' relay|index:'id' %}"
+              hx-target="#relays"
+              hx-swap="outerHTML"
+              class="button is-small has-background-danger has-text-white">
+              <span class="icon" data-tooltip="Delete">
+                <i class="fa-solid fa-xmark" aria-hidden="true"></i>
+              </span>
+            </a>
+          </td>
+          <td>
+            <a
+              hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
+              hx-post="{% url 'threshold_irc_network_relay_provision' relay|index:'net' relay|index:'id' %}"
+              hx-target="#relays"
+              hx-swap="outerHTML"
+              class="button is-small has-background-info has-text-white">
+              <span class="icon" data-tooltip="Provision">
                 <i class="fa-solid fa-wrench" aria-hidden="true"></i>
               </span>
-            </th>
-          </thead>
-          <tbody>
-            {% for relay in relays %}
-              <tr>
-                <td>{{ relay.id }}</td>
-                <td>
-                  {% if relay.registered %}
-                    <span class="icon">
-                      <i class="fa-solid fa-check" aria-hidden="true"></i>
-                    </span>
-                  {% else %}
-                    <span class="icon">
-                      <i class="fa-solid fa-xmark" aria-hidden="true"></i>
-                    </span>
-                  {% endif %}
-                </td>
-                <td>
-                  {% if relay.enabled %}
-                    <span class="icon">
-                      <i class="fa-solid fa-check" aria-hidden="true"></i>
-                    </span>
-                  {% else %}
-                    <span class="icon">
-                      <i class="fa-solid fa-xmark" aria-hidden="true"></i>
-                    </span>
-                  {% endif %}
-                </td>
-                <td>
-                  {{ relay.chans }}
-                </td>
-                <td>
-                  {{ relay.nick }}
-                </td>
-                <td>
-                  <div class="buttons">
-                    {% if relay.enabled %}
-                      <button
-                        hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
-                        hx-get="{% url 'threshold_irc_network_relay_status' relay|index:'net' relay|index:'id' 0 %}"
-                        hx-target="#relays"
-                        hx-swap="outerHTML"
-                        class="button is-danger is-small">
-                        <span class="icon" data-tooltip="Disable">
-                          <i class="fa-solid fa-wifi-slash" aria-hidden="true"></i>
-                        </span>
-                      </button>
-                    {% else %}
-                      <button
-                        hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
-                        hx-get="{% url 'threshold_irc_network_relay_status' relay|index:'net' relay|index:'id' 1 %}"
-                        hx-target="#relays"
-                        hx-swap="outerHTML"
-                        class="button is-success is-small">
-                        <span class="icon" data-tooltip="Enable">
-                          <i class="fa-solid fa-wifi" aria-hidden="true"></i>
-                        </span>
-                      </button>
-                    {% endif %}
-                    <button
-                      hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
-                      hx-delete="{% url 'threshold_irc_network_relay_del' relay|index:'net' relay|index:'id' %}"
-                      hx-target="#relays"
-                      hx-swap="outerHTML"
-                      class="button is-danger is-small">
-                      <span class="icon" data-tooltip="Delete">
-                        <i class="fa-solid fa-xmark" aria-hidden="true"></i>
-                      </span>
-                    </button>
-                  </div>
-                </td>
-              </tr>
-            {% endfor %}
-          </tbody>
-        </table>
-      </div>
-    </div>
-  {% endif %}
+            </a>
+          </td>
+          <td>
+            <a
+              hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
+              hx-post="{% url 'threshold_irc_network_relay_auth' relay|index:'net' relay|index:'id' %}"
+              hx-target="#relays"
+              hx-swap="outerHTML"
+              class="button is-small has-background-info has-text-white">
+              <span class="icon" data-tooltip="Enable authentication">
+                <i class="fa-solid fa-passport" aria-hidden="true"></i>
+              </span>
+            </a>
+          </td>
+          <td></td>
+          <td>
+            {% if relay.enabled %}
+              <a
+                hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
+                hx-get="{% url 'threshold_irc_network_relay_status' relay|index:'net' relay|index:'id' 0 %}"
+                hx-target="#relays"
+                hx-swap="outerHTML"
+                class="button is-small has-background-warning">
+                <span class="icon" data-tooltip="Disable">
+                  <i class="fa-solid fa-wifi-slash" aria-hidden="true"></i>
+                </span>
+              </a>
+            {% else %}
+              <a
+                hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
+                hx-get="{% url 'threshold_irc_network_relay_status' relay|index:'net' relay|index:'id' 1 %}"
+                hx-target="#relays"
+                hx-swap="outerHTML"
+                class="button is-small has-background-success has-text-white">
+                <span class="icon" data-tooltip="Enable">
+                  <i class="fa-solid fa-wifi" aria-hidden="true"></i>
+                </span>
+              </a>
+            {% endif %}
+          </td>
+          <td></td>
+          <td></td>
+          <td></td>
+          <td>
+            <a
+              hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
+              hx-post="{% url 'modal_context' %}"
+              hx-vals='{"net": "{{ net }}",
+              "num": "{{ relay.id }}",
+              "source": "irc",
+              "channel": "{{ sinst.entity }}",
+              "time": "None",
+              "date": "None",
+              "index": "internal",
+              "type": "auth",
+              "mtype": "None",
+              "nick": "{{ sinst.entity }}",
+              "dedup": "on"}'
+              hx-target="#modals-here"
+              hx-trigger="click"
+              class="button is-small has-background-info has-text-white">
+              <span class="icon has-tooltip-left" data-tooltip="Auth ({{ sinst.entity }})">
+                <i class="fa-solid fa-signature" aria-hidden="true"></i>
+              </span>
+            </a>
+          </td>
+        </tr>
+      {% endfor %}
+    </tbody>
+  </table>
+  <script>
+    var modal_event = new Event('restore-relay-scroll');
+    document.dispatchEvent(modal_event);
+  </script>
+  {% include 'manage/threshold/partials/notify.html' %}
 </div>

core/templates/manage/threshold/irc/network/stats.html

@@ -0,0 +1,27 @@
+<div id="stats">
+  {% include 'manage/threshold/partials/notify.html' %}
+  {% if list is not None %}
+    <div class="content" style="max-height: 30em; overflow: auto;">
+      <div class="table-container">
+        <table class="table is-fullwidth is-hoverable">
+          <thead>
+            <th>attribute</th>
+            <th>value</th>
+          </thead>
+          <tbody>
+            {% for key, item in list.items %}
+              <tr>
+                <th>{{ key }}</th>
+                <td>
+                  {% if item is not None %}
+                    {{ item }}
+                  {% endif %}
+                </td>
+              </tr>
+            {% endfor %}
+          </tbody>
+        </table>
+      </div>
+    </div>
+  {% endif %}
+</div>

core/templates/manage/threshold/irc/overview/actions.html

@@ -30,6 +30,9 @@
     </button>
     <button
       hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
+      hx-get="{% url 'threshold_irc_actions_registration' %}"
+      hx-trigger="click"
+      hx-target="#modals-here"
       class="button is-info">
       <span class="icon-text">
         <span class="icon">

core/templates/manage/threshold/irc/overview/alerts.html

@@ -1,94 +0,0 @@
-<div id="alerts">
-  {% include 'manage/threshold/partials/notify.html' %}
-  {% if alerts is not None %}
-    <div class="icons">
-      <button
-        class="button is-small">
-        <span class="icon" data-tooltip="Conn">
-          <i class="fa-solid fa-cloud-exclamation"></i>
-        </span>
-      </button>
-      <button
-        class="button is-small">
-        <span class="icon" data-tooltip="Highlight">
-          <i class="fa-solid fa-square-quote"></i>
-        </span>
-      </button>
-
-      <button
-        class="button is-small">
-        <span class="icon" data-tooltip="ZNC">
-          <i class="fa-brands fa-unity"></i>
-        </span>
-      </button>
-
-      <button
-        class="button is-small">
-        <span class="icon" data-tooltip="Query">
-          <i class="fa-solid fa-inbox"></i>
-        </span>
-      </button>
-
-      <button class="button is-small">
-        <span class="icon" data-tooltip="Self">
-          <i class="fa-solid fa-message-bot"></i>
-        </span>
-      </button>
-
-    </div>
-    <div class="content" style="max-height: 30em; overflow: auto;">
-
-      <div class="table-container">
-        <table class="table is-fullwidth is-hoverable">
-          <thead>
-            <th>ts</th>
-            <th>name</th>
-            <th>type</th>
-            <th>msg</th>
-          </thead>
-          <tbody>
-            {% for alert in alerts %}
-              <tr>
-                <td>
-                  <p>{{ alert.date }}</p>
-                  <p>{{ alert.time }}</p>
-                </td>
-                <td>
-                  {{ alert.net }}/{{ alert.num }}
-                </td>
-                <td>
-                  {% if alert.type == 'conn' %}
-                    <span class="icon" data-tooltip="Conn">
-                      <i class="fa-solid fa-cloud-exclamation"></i>
-                    </span>
-                  {% elif alert.type == 'highlight' %}
-                    <span class="icon" data-tooltip="Highlight">
-                      <i class="fa-solid fa-square-quote"></i>
-                    </span>
-                  {% elif alert.type == 'znc' %}
-                    <span class="icon" data-tooltip="ZNC">
-                      <i class="fa-brands fa-unity"></i>
-                    </span>
-                  {% elif alert.type == 'query' %}
-                    <span class="icon" data-tooltip="Query">
-                      <i class="fa-solid fa-inbox"></i>
-                    </span>
-                  {% elif alert.type == 'self' %}
-                    <span class="icon" data-tooltip="Self">
-                      <i class="fa-solid fa-message-bot"></i>
-                    </span>
-                  {% else %}
-                    {{ alert.type }}
-                  {% endif %}
-                </td>
-                <td class="wrap" style="max-width: 10em">
-                  {{ alert.msg }}
-                </td>
-              </tr>
-            {% endfor %}
-          </tbody>
-        </table>
-      </div>
-    </div>
-  {% endif %}
-</div>

core/templates/manage/threshold/irc/overview/modals/add-network.html

@@ -2,7 +2,7 @@
 {% load static %}
 
 <script src="{% static 'modal.js' %}"></script>
-<div class="modal is-active is-clipped">
+<div id="modal" class="modal is-active is-clipped">
   <div class="modal-background"></div>
   <div class="modal-content">
     <div class="box">
@@ -60,7 +60,7 @@
                 </div>
               </div>
             </div>
-          </div> 
+          </div>
         </div>
         <button
           type="button"
@@ -68,9 +68,9 @@
           Cancel
         </button>
         <button type="submit" class="button is-info modal-close-button">Submit</button>
-        <script>activateButtons();</script>
+        {# <script>activateButtons();</script> #}
       </form>
-  
+
     </div>
     <button class="modal-close is-large" aria-label="close"></button>
   </div>

core/templates/manage/threshold/irc/overview/modals/edit-aliases.html

@@ -3,7 +3,7 @@
 {% load nsep %}
 
 <script src="{% static 'modal.js' %}"></script>
-<div class="modal is-active is-clipped">
+<div id="modal" class="modal is-active is-clipped">
   <div class="modal-background"></div>
   <div class="modal-content">
     <div class="box">
@@ -45,10 +45,10 @@
                     </td>
                   </tr>
                 {% endfor %}
-    
+
               </tbody>
             </table>
-    
+
           </div>
         </div>
         <button
@@ -59,7 +59,7 @@
         <button type="submit" class="button is-info modal-close-button">Submit</button>
         <script>activateButtons();</script>
       </form>
-  
+
     </div>
     <button class="modal-close is-large" aria-label="close"></button>
   </div>

core/templates/manage/threshold/irc/overview/networks.html

@@ -4,15 +4,28 @@
     <table class="table is-fullwidth is-hoverable">
       <thead>
         <th>net</th>
-        <th>relays</th>
         <th>
-          <span class="icon">
+          <span class="icon has-tooltip-bottom" data-tooltip="Relays">
+            <i class="fa-brands fa-unity"></i>
+          </span>
+        </th>
+        <th>
+          <span class="icon has-tooltip-bottom" data-tooltip="Active">
+            <i class="fa-solid fa-signal-bars-good"></i>
+          </span>
+        </th>
+        <th>
+          <span class="icon has-tooltip-bottom" data-tooltip="Channels">
             <i class="fa-solid fa-hashtag"></i>
           </span>
         </th>
-        <th>records</th>
         <th>
-          <span class="icon">
+          <span class="icon has-tooltip-bottom" data-tooltip="Records">
+            <i class="fa-solid fa-album"></i>
+          </span>
+        </th>
+        <th>
+          <span class="icon has-tooltip-bottom" data-tooltip="Actions">
             <i class="fa-solid fa-wrench" aria-hidden="true"></i>
           </span>
         </th>
@@ -21,22 +34,25 @@
         <tr>
           <th><a href="{% url 'threshold_irc_network' key %}">{{ key }}</a></th>
           <td>
-            
-            <span class="icon">
-              <i class="fa-brands fa-unity"></i>
-            </span>
+
+
             {{ net.relays }}
           </td>
           <td>
-            <span class="icon">
-              <i class="fa-solid fa-hashtag"></i>
-            </span>
+            {% if net.active %}
+              <span class="icon has-text-success">
+                <i class="fa-solid fa-check" aria-hidden="true"></i>
+              </span>
+            {% else %}
+              <span class="icon has-text-danger">
+                <i class="fa-solid fa-xmark" aria-hidden="true"></i>
+              </span>
+            {% endif %}
+          </td>
+          <td>
             {{ net.channels }}
           </td>
           <td>
-            <span class="icon">
-              <i class="fa-solid fa-album"></i>
-            </span>
             {{ net.records }}
           </td>
           <td>
@@ -45,7 +61,7 @@
               hx-delete="{% url 'threshold_irc_network_del' key %}"
               hx-target="#networks"
               hx-swap="outerHTML"
-              class="button is-danger is-small">
+              class="button is-small is-danger">
               <span class="icon" data-tooltip="Delete">
                 <i class="fa-solid fa-xmark" aria-hidden="true"></i>
               </span>

core/templates/manage/threshold/irc/overview/overview.html

@@ -37,15 +37,6 @@
     hx-swap="outerHTML">
   </div>
 
-  <div
-    style="display: none;"
-    hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
-    hx-get="{% url 'threshold_irc_overview_alerts' %}"
-    hx-trigger="load"
-    hx-target="#alerts"
-    hx-swap="outerHTML">
-  </div>
-
   <div class="columns">
     <div class="column">
       <div class="box">
@@ -71,6 +62,7 @@
     <div class="column">
       <div class="box">
         <div id="alerts">
+          Alerts here
         </div>
       </div>
     </div>

core/templates/modals/context.html

@@ -0,0 +1,121 @@
+{% extends 'wm/modal.html' %}
+
+{% load index %}
+{% load static %}
+
+{% block scripts %}
+  <script>
+    document.addEventListener("restore-modal-scroll", function(event) {
+      var modalContent = document.getElementsByClassName("modal-content")[0];
+      var maxScroll = modalContent.scrollHeight - modalContent.offsetHeight;
+      var scrollpos = localStorage.getItem('scrollpos_modal_content');
+      if (scrollpos == 'BOTTOM') {
+        modalContent.scrollTop = maxScroll;
+      } else if (scrollpos) {
+        modalContent.scrollTop = scrollpos;
+      };
+    });
+
+    document.addEventListener("htmx:beforeSwap", function(event) {
+      var modalContent = document.getElementsByClassName("modal-content")[0];
+      var scrollpos = modalContent.scrollTop;
+      if(modalContent.scrollTop === (modalContent.scrollHeight - modalContent.offsetHeight)) {
+        localStorage.setItem('scrollpos_modal_content', 'BOTTOM');
+      } else {
+        localStorage.setItem('scrollpos_modal_content', scrollpos);
+      }
+
+    });
+  </script>
+{% endblock %}
+
+{% block styles %}
+  <style>
+    #tab-content-{{ unique }} div {
+      display: none;
+    }
+
+    #tab-content-{{ unique }} div.is-active {
+      display: block;
+    }
+  </style>
+{% endblock %}
+
+{% block modal_content %}
+  {% include 'partials/notify.html' %}
+  <div class="tabs is-toggle is-fullwidth is-info" id="tabs-{{ unique }}">
+    <ul>
+      <li class="is-active" data-tab="1">
+        <a>
+          <span class="icon is-small"><i class="fa-solid fa-message-arrow-down"></i></span>
+          <span>Scrollback</span>
+        </a>
+      </li>
+      <li data-tab="2">
+        <a>
+          <span class="icon is-small"><i class="fa-solid fa-messages"></i></span>
+          <span>Context</span>
+        </a>
+      </li>
+      <li data-tab="3">
+        <a>
+          <span class="icon is-small"><i class="fa-solid fa-message"></i></span>
+          <span>Message</span>
+        </a>
+      </li>
+      <li data-tab="4">
+        <a>
+          <span class="icon is-small"><i class="fa-solid fa-asterisk"></i></span>
+          <span>Info</span>
+        </a>
+      </li>
+    </ul>
+  </div>
+  <div id="tab-content-{{ unique }}">
+    <div class="is-active" data-content="1">
+      <h4 class="subtitle is-4">Scrollback of {{ channel }} on {{ net }}{% if num is not None %}{{ num }}{% endif %}</h4>
+      {% include 'partials/context_table.html' %}
+      {% if user.is_superuser and source == 'irc' %}
+        <form method="PUT">
+          <article class="field has-addons">
+            <article class="control is-expanded has-icons-left">
+              <input id="context-input" name="msg" class="input" type="text" placeholder="Type your message here">
+              <span class="icon is-small is-left">
+                <i class="fas fa-magnifying-glass"></i>
+              </span>
+            </article>
+            <article class="control">
+              <article class="field">
+                <button
+                  id="search"
+                  class="button is-info is-fullwidth"
+                  hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
+                  hx-put="{% url 'threshold_irc_msg' net num %}"
+                  hx-vals='{"channel": "{{ channel }}", "nick": "{{ nick }}"}'
+                  hx-trigger="click"
+                  hx-target="#context-input"
+                  hx-swap="outerHTML">
+                  Send
+                </button>
+              </article>
+            </article>
+          </article>
+        </form>
+      {% endif %}
+
+    </div>
+    <div data-content="2">
+      <h4 class="subtitle is-4">Scrollback of {{ channel }} on {{ net }}{{ num }} around {{ ts }}</h4>
+      Context
+    </div>
+    <div data-content="3">
+      <h4 class="subtitle is-4">Message details</h4>
+      Message deetails
+    </div>
+    <div data-content="4">
+      <h4 class="subtitle is-4">Information about {{ channel }} on {{ net }}{{ num }}</h4>
+      info
+    </div>
+  </div>
+  <script>initTabs("{{ unique }}");</script>
+{% endblock %}

core/templates/modals/drilldown.html

@@ -1,108 +1,5 @@
-{% load index %}
-{% load static %}
+{% extends 'wm/modal.html' %}
 
-<script src="{% static 'modal.js' %}"></script>
-<link rel ="stylesheet" href="{% static 'tabs.css' %}">
-<script src="{% static 'tabs.js' %}"></script>
-<div class="modal is-active is-clipped">
-  <div class="modal-background"></div>
-  <div class="modal-content">
-    <div class="box">
-      <div class="tabs is-toggle is-fullwidth is-info" id="tabs">
-        <ul>
-          <li class="is-active" data-tab="1">
-            <a>
-              <span class="icon is-small"><i class="fa-solid fa-user"></i></span>
-              <span>Channels</span>
-            </a>
-          </li>
-          <li data-tab="2">
-            <a>
-              <span class="icon is-small"><i class="fa-solid fa-hashtag"></i></span>
-              <span>Users</span>
-            </a>
-          </li>
-          <li data-tab="3">
-            <a>
-              <span class="icon is-small"><i class="fa-solid fa-people"></i></span>
-              <span>Intersection</span>
-            </a>
-          </li>
-          <li data-tab="4">
-            <a>
-              <span class="icon is-small"><i class="fa-solid fa-hashtag"></i></span>
-              <span>Intersection</span>
-            </a>
-          </li>
-        </ul>
-      </div>
-      <div id="tab-content">
-        <div class="is-active" data-content="1">
-          <h4 class="subtitle is-4">Channels for {{ nick }} on {{ net }}</h4>
-          {% for channel in chans %}
-            <a class="panel-block is-active">
-              <span class="panel-icon">
-                <i class="fa-solid fa-hashtag" aria-hidden="true"></i>
-              </span>
-              {{ channel }}
-              {% if nick in num_chans %}
-                <span class="tag">
-                  {{ num_users|index:channel }}
-                </span>
-              {% endif %}
-            </a>
-          {% endfor %}
-        </div>
-        <div data-content="2">
-          <h4 class="subtitle is-4">Users on {{ channel }} for {{ net }}</h4>
-          {% for user in users %}
-            <a class="panel-block is-active">
-              <span class="panel-icon">
-                <i class="fa-solid fa-user" aria-hidden="true"></i>
-              </span>
-              {{ user }}
-              {% if channel in num_users %}
-                <span class="tag">
-                  {{ num_chans|index:user }}
-                </span>
-              {% endif %}
-            </a>
-          {% endfor %}
-        </div>
-        <div data-content="3">
-          <h4 class="subtitle is-4">Users sharing channels with {{ nick }} on {{ net }}</h4>
-          {% for user in inter_users %}
-            <a class="panel-block is-active">
-              <span class="panel-icon">
-                <i class="fa-solid fa-user" aria-hidden="true"></i>
-              </span>
-              {{ user }}
-              {% if channel in num_users %}
-                <span class="tag">
-                  {{ num_chans|index:user }}
-                </span>
-              {% endif %}
-            </a>
-          {% endfor %}
-        </div>
-        <div data-content="4">
-          <h4 class="subtitle is-4">Channels sharing users with {{ channel }} on {{ net }}</h4>
-          {% for channel in inter_chans %}
-            <a class="panel-block is-active">
-              <span class="panel-icon">
-                <i class="fa-solid fa-hashtag" aria-hidden="true"></i>
-              </span>
-              {{ channel }}
-              {% if nick in num_chans %}
-                <span class="tag">
-                  {{ num_users|index:channel }}
-                </span>
-              {% endif %}
-            </a>
-          {% endfor %}
-        </div>
-      </div>
-    </div>
-    <button class="modal-close is-large" aria-label="close"></button>
-  </div>
-</div>
+{% block modal_content %}
+  {% include 'window-content/drilldown.html' %}
+{% endblock %}

core/templates/partials/context-input.html

@@ -0,0 +1 @@
+<input id="context-input" name="msg" class="input is-{{ class }}" type="text" placeholder="Type your message here">

core/templates/partials/context_table.html

@@ -0,0 +1,177 @@
+<article class="table-container" id="modal-context-table">
+  <table class="table is-fullwidth">
+    <thead>
+      <th></th>
+      <th></th>
+      <th></th>
+    </thead>
+    <tbody>
+      {% for item in object_list %}
+        {% if item.type == 'control' %}
+          <tr>
+            <td></td>
+            <td>
+              <span class="icon has-text-grey" data-tooltip="Hidden">
+                <i class="fa-solid fa-file-slash"></i>
+              </span>
+            </td>
+            <td>
+              <p class="has-text-grey">Hidden {{ item.hidden }} similar result{% if item.hidden > 1%}s{% endif %}</p>
+            </td>
+          </tr>
+        {% else %}
+          <tr>
+            <td>{{ item.time }}</td>
+            <td>
+              {% if item.type != 'znc' and item.type != 'self' and query is not True %}
+                <article class="nowrap-parent">
+                  <article class="nowrap-child">
+                    {% if item.type == 'msg' %}
+                      <span class="icon" data-tooltip="Message">
+                        <i class="fa-solid fa-message"></i>
+                      </span>
+                    {% elif item.type == 'join' %}
+                      <span class="icon" data-tooltip="Join">
+                        <i class="fa-solid fa-person-to-portal"></i>
+                      </span>
+                    {% elif item.type == 'part' %}
+                      <span class="icon" data-tooltip="Part">
+                        <i class="fa-solid fa-person-from-portal"></i>
+                      </span>
+                    {% elif item.type == 'quit' %}
+                      <span class="icon" data-tooltip="Quit">
+                        <i class="fa-solid fa-circle-xmark"></i>
+                      </span>
+                    {% elif item.type == 'kick' %}
+                      <span class="icon" data-tooltip="Kick">
+                        <i class="fa-solid fa-user-slash"></i>
+                      </span>
+                    {% elif item.type == 'nick' %}
+                      <span class="icon" data-tooltip="Nick">
+                        <i class="fa-solid fa-signature"></i>
+                      </span>
+                    {% elif item.type == 'mode' %}
+                      <span class="icon" data-tooltip="Mode">
+                        <i class="fa-solid fa-gear"></i>
+                      </span>
+                    {% elif item.type == 'action' %}
+                      <span class="icon" data-tooltip="Action">
+                        <i class="fa-solid fa-exclamation"></i>
+                      </span>
+                    {% elif item.type == 'notice' %}
+                      <span class="icon" data-tooltip="Notice">
+                        <i class="fa-solid fa-message-code"></i>
+                      </span>
+                    {% elif item.type == 'conn' %}
+                      <span class="icon" data-tooltip="Connection">
+                        <i class="fa-solid fa-cloud-exclamation"></i>
+                      </span>
+                    {% elif item.type == 'znc' %}
+                      <span class="icon" data-tooltip="ZNC">
+                        <i class="fa-brands fa-unity"></i>
+                      </span>
+                    {% elif item.type == 'query' %}
+                      <span class="icon" data-tooltip="Query">
+                        <i class="fa-solid fa-message"></i>
+                      </span>
+                    {% elif item.type == 'highlight' %}
+                      <span class="icon" data-tooltip="Highlight">
+                        <i class="fa-solid fa-exclamation"></i>
+                      </span>
+                    {% elif item.type == 'who' %}
+                      <span class="icon" data-tooltip="Who">
+                        <i class="fa-solid fa-passport"></i>
+                      </span>
+                    {% elif item.type == 'topic' %}
+                      <span class="icon" data-tooltip="Topic">
+                        <i class="fa-solid fa-sign"></i>
+                      </span>
+                    {% else %}
+                      {{ item.type }}
+                    {% endif %}
+                    {% if item.online is True %}
+                      <span class="icon has-text-success has-tooltip-success" data-tooltip="Online">
+                        <i class="fa-solid fa-circle"></i>
+                      </span>
+                    {% elif item.online is False %}
+                      <span class="icon has-text-danger has-tooltip-danger" data-tooltip="Offline">
+                        <i class="fa-solid fa-circle"></i>
+                      </span>
+                    {% else %}
+                      <span class="icon has-text-warning has-tooltip-warning" data-tooltip="Unknown">
+                        <i class="fa-solid fa-circle"></i>
+                      </span>
+                    {% endif %}
+                    {% if item.src == 'irc' %}
+                      <a
+                        hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
+                        hx-post="{% url 'modal_drilldown' %}"
+                        hx-vals='{"net": "{{ item.net }}", "nick": "{{ item.nick }}", "channel": "{{ item.channel }}"}'
+                        hx-target="#modals-here"
+                        hx-trigger="click"
+                        class="has-text-black">
+                        <span class="icon" data-tooltip="Open drilldown modal">
+                          <i class="fa-solid fa-album"></i>
+                        </span>
+                      </a>
+                    {% endif %}
+                  </article>
+                  <a class="nowrap-child has-text-link is-underlined" onclick="populateSearch('nick', '{{ item.nick|escapejs }}')">
+                    {{ item.nick }}
+                  </a>
+                  {% if item.num_chans != '—' %}
+                    <article class="nowrap-child">
+                      <span class="tag">
+                        {{ item.num_chans }}
+                      </span>
+                    </article>
+                  {% endif %}
+                </article>
+              {% endif %}
+              {% if item.type == 'self' %}
+                <span class="icon has-text-primary" data-tooltip="You">
+                  <i class="fa-solid fa-message-check"></i>
+                </span>
+              {% elif item.type == 'znc' %}
+                <span class="icon has-text-info" data-tooltip="ZNC">
+                  <i class="fa-brands fa-unity"></i>
+                </span>
+              {% elif query %}
+                <span class="icon has-text-info" data-tooltip="Auth">
+                  <i class="fa-solid fa-passport"></i>
+                </span>
+              {% endif %}
+            </td>
+            <td class="wrap">{{ item.msg }}</td>
+          </tr>
+        {% endif %}
+      {% endfor %}
+    </tbody>
+  </table>
+  {% if object_list %}
+    <div
+      class="modal-refresh"
+      style="display: none;"
+      hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
+      hx-post="{% url 'modal_context_table' %}"
+      hx-vals='{"net": "{{ net }}",
+      "num": "{{ num }}",
+      "source": "{{ source }}",
+      "channel": "{{ channel }}",
+      "time": "{{ time }}",
+      "date": "{{ date }}",
+      "index": "{{ index }}",
+      "type": "{{ type }}",
+      "mtype": "{{ mtype }}",
+      "nick": "{{ nick }}",
+      "dedup": "{{ params.dedup }}"}'
+      hx-target="#modal-context-table"
+      hx-trigger="every 5s">
+    </div>
+  {% endif %}
+</article>
+
+<script>
+  var modal_event = new Event('restore-modal-scroll');
+  document.dispatchEvent(modal_event);
+</script>

core/templates/partials/notify-alt.html

@@ -0,0 +1,5 @@
+{% if message is not None %}
+  <main class="notification is-{{ class }}" hx-ext="remove-me" remove-me="3s">
+    {{ message }}
+  </main>
+{% endif %}

core/templates/partials/notify.html

@@ -2,4 +2,4 @@
   <div class="notification is-{{ class }}" hx-ext="remove-me" remove-me="3s">
     {{ message }}
   </div>
-{% endif %}
+{% endif %}

core/templates/ui/drilldown/drilldown.html

@@ -1,8 +1,28 @@
 {% extends "base.html" %}
 {% load static %}
 {% load joinsep %}
-{% block content %}
+{% block outer_content %}
+  {% if params.modal == 'context' %}
+    <div
+      style="display: none;"
+      hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
+      hx-post="{% url 'modal_context' %}"
+      hx-vals='{"net": "{{ params.net|escapejs }}",
+      "num": "{{ params.num|escapejs }}",
+      "source": "{{ params.source|escapejs }}",
+      "channel": "{{ params.channel|escapejs }}",
+      "time": "{{ params.time|escapejs }}",
+      "date": "{{ params.date|escapejs }}",
+      "index": "{{ params.index }}",
+      "type": "{{ params.type|escapejs }}",
+      "mtype": "{{ params.mtype|escapejs }}",
+      "nick": "{{ params.nick|escapejs }}"}'
+      hx-target="#modals-here"
+      hx-trigger="load">
+    </div>
+  {% endif %}
   <script src="{% static 'js/chart.js' %}"></script>
+  <script src="{% static 'tabs.js' %}"></script>
   <script>
     function setupTags() {
       var inputTags = document.getElementById('tags');
@@ -20,359 +40,124 @@
         } catch {
           var value = spl[1];
         }
-        populateSearch(field, value);
         return `${field}: ${value}`;
       });
       inputTags.BulmaTagsInput().on('after.remove', function(item) {
         var spl = item.split(": ");
         var field = spl[0];
-        try {
-          var value = JSON.parse(spl[1]);
-        } catch {
-          var value = spl[1].trim();
-        }
-        populateSearch(field, value);
+        var value = spl[1].trim();
       });
     }
     function populateSearch(field, value) {
-      var queryElement = document.getElementById('query');
-      var present = true;
-      if (present == true) {
-        var combinations = [`${field}: "${value}"`,
-                            `${field}: "${value}"`,
-                            `${field}: ${value}`,
-                            `${field}:${value}`,
-                            `${field}:"${value}"`];
-        var toAppend = ` AND ${field}: "${value}"`;
-        // var toRemove = `${field}: "${value}"`;
-        // var tagText = `${field}: ${value}`;
-      } else {
-        var combinations = [`NOT ${field}: "${value}"`,
-                            `NOT ${field}: "${value}"`,
-                            `NOT ${field}: ${value}`,
-                            `NOT ${field}:${value}`,
-                            `NOT ${field}:"${value}"`];
-        // var toAppend = ` AND NOT ${field}: "${value}"`;
-        // var toRemove = `NOT ${field}: "${value}"`;
-      }
-      var contains = combinations.some(elem => queryElement.value.includes(elem));
-      if (!contains) {
-          queryElement.value+=toAppend;
-      } else {
-        for (var index in combinations) {
-          combination = combinations[index];
-          queryElement.value = queryElement.value.replaceAll("AND "+combination, "");
-          queryElement.value = queryElement.value.replaceAll(combination, "");
-        }
-          // queryElement.value = queryElement.value.replaceAll(toAppend, "");
-          // queryElement.value = queryElement.value.replaceAll(toRemove, "");
-
-      }
-      // if (!queryElement.value.includes(toAppend) && !queryElement.value.includes(toRemove)) {
-      //   queryElement.value+=toAppend;
-      // } else {
-      //     queryElement.value = queryElement.value.replaceAll(toAppend, "");
-      //     queryElement.value = queryElement.value.replaceAll(toRemove, "");
-
-      // }
-      if (field == "src") {
-        document.getElementById("source").selectedIndex = 0;
-      }
-      if (queryElement.value.startsWith(" AND ")) {
-        queryElement.value = queryElement.value.replace(" AND ", "");
-      }
-      if (queryElement.value.startsWith("AND ")) {
-        queryElement.value = queryElement.value.replace("AND ", "");
-      }
-      htmx.trigger("#search", "click");
+      var inputTags = document.getElementById('tags');
+      inputTags.BulmaTagsInput().add(field+": "+value);
+      //htmx.trigger("#search", "click");
     }
   </script>
-  <div>
-    <form method="POST" hx-post="{% url 'home' %}"
-          hx-trigger="change"
-          hx-target="#results"
-          hx-swap="innerHTML"
-          hx-indicator="#spinner">
-      {% csrf_token %}
 
-      <div class="columns">
-        <div class="column">
-          <div class="field has-addons">
-            <div class="control is-expanded has-icons-left">
-              <input
-                hx-post="{% url 'home' %}"
-                hx-trigger="keyup changed delay:200ms"
-                hx-target="#results"
-                hx-swap="innerHTML" id="query" name="query" value="{{ params.query }}" class="input" type="text" placeholder="msg: science AND nick: BillNye AND channel: #science">
-              <span class="icon is-small is-left">
-                <i class="fas fa-magnifying-glass"></i>
-              </span>
-            </div>
-            <div class="control">
-              <div class="field">
-                <button
-                  id="search"
-                  class="button is-info is-fullwidth"
-                  hx-post="{% url 'home' %}"
-                  hx-trigger="click"
-                  hx-target="#results"
-                  hx-swap="innerHTML">
-                  Search
-                </button>
-              </div>
-            </div>
-          </div>
-        </div>
-        <div class="column is-3">
-          <div class="nowrap-parent">
-            <div
-              data-script="on click toggle .is-hidden on #options"
-              class="button is-light has-text-link is-right nowrap-child">
-              Options
-            </div>
-            <div class="nowrap-child">
-              <span id="spinner" class="button is-light has-text-link is-loading htmx-indicator">Static</span>
-            </div>
-          </div>
-        </div>
+  <div class="grid-stack" id="grid-stack-main">
+    <div class="grid-stack-item" gs-w="7" gs-h="10" gs-y="0" gs-x="1">
+      <div class="grid-stack-item-content">
+        <nav class="panel">
+          <p class="panel-heading" style="padding: .2em; line-height: .5em;">
+            <i class="fa-solid fa-arrows-up-down-left-right has-text-grey-light"></i>
+            Search
+          </p>
+          <article class="panel-block is-active">
+            {% include 'ui/drilldown/search_partial.html' %}
+          </article>
+        </nav>
       </div>
-      <div id="options" class="box is-hidden">
-        <div class="columns is-multiline">
-          <div class="column is-narrow">
-            <div class="field has-addons">
-              <div class="control has-icons-left">
-                <span class="select">
-                  <select name="size">
-                    {% for size in sizes %}
-                      {% if size == params.size %}
-                        <option selected value="{{ size }}">{{ size }}</option>
-                      {% else %}
-                        <option value="{{ size }}">{{ size }}</option>
-                      {% endif %}
-                    {% endfor %}
-                  </select>
-                  <span class="icon is-small is-left">
-                    <i class="fas fa-magnifying-glass"></i>
-                  </span>
-                </span>
-              </div>
-              <p class="control">
-                <a class="button is-static">
-                  results
-                </a>
-              </p>
-            </div>
-          </div>
-          <div class="column is-narrow">
-            <div class="field has-addons">
-              <div class="control has-icons-left">
-                <span class="select">
-                  <select id="source" name="source">
-                    {% if params.source == 'irc' %}
-                      <option selected value="irc">IRC</option>
-                    {% else %}
-                      <option value="irc">IRC</option>
-                    {% endif %}
-
-                    {% if params.source == 'dis' %}
-                      <option selected value="dis">Discord</option>
-                    {% else %}
-                      <option value="dis">Discord</option>
-                    {% endif %}
-
-                    {% if params.source == None %}
-                      <option selected value="all">All</option>
-                    {% elif params.source == 'all' %}
-                      <option selected value="all">All</option>
-                    {% else %}
-                      <option value="all">All</option>
-                    {% endif %}
-                    
-                  </select>
-                  <span class="icon is-small is-left">
-                    <i class="fas fa-magnifying-glass"></i>
-                  </span>
-                </span>
-              </div>
-              <p class="control">
-                <a class="button is-static">
-                  source
-                </a>
-              </p>
-            </div>
-          </div>
-          <div class="column is-narrow">
-            <div id="sentiment">
-              <div class="field has-addons">
-                <div class="control">
-                  <input
-                    {% if params.check_sentiment != "on" %}
-                      disabled="undefined"
-                    {% endif %}
-                    name="sentiment" id="sliderWithValue" class="slider has-output-tooltip is-fullwidth" min="-1" max="1"
-                    {% if params.sentiment == None %}
-                      value="0"
-                    {% else %}
-                      value="{{ params.sentiment }}"
-                    {% endif %}
-                    step="0.05" type="range">
-                  <output for="sliderWithValue" class="slider-output">
-                    {% if params.sentiment == None %}
-                      0
-                    {% else %}
-                      {{ params.sentiment }}
-                    {% endif %}
-                  </output>
-                  <script>bulmaSlider.attach();</script>
-                </div>
-                <p class="control">
-                  <a class="button is-static">
-                    sentiment
-                  </a>
-                </p>
-              </div>
-              <div class="control">
-                <label class="radio button has-text-link">
-                  
-                  <input type="radio"
-                         value="below"
-                         {% if params.sentiment_method == 'below' %}
-                           checked
-                         {% endif %}
-                         name="sentiment_method">
-                  <span class="icon" data-tooltip="Below">
-                    <i class="fa-solid fa-face-frown"></i>
-                  </span>
-                </label>
-                <label class="radio button has-text-link is-hidden">
-
-                  <input type="radio"
-                         value="exact"
-                         {% if params.sentiment_method == 'exact' %}
-                           checked
-                         {% endif %}
-                         name="sentiment_method">
-                  <span class="icon" data-tooltip="Exact">
-                    <i class="fa-solid fa-face-smile"></i>
-                  </span>
-                </label>
-                <label class="radio button has-text-link">
-                  <input type="radio"
-                         value="above"
-                         {% if params.sentiment_method == 'above' %}
-                           checked
-                         {% endif %}
-                         name="sentiment_method">
-                  <span class="icon" data-tooltip="Above">
-                    <i class="fa-solid fa-face-smile"></i>
-                  </span>
-                </label>
-                <label class="radio button has-text-link">
-                  <input type="radio"
-                         value="nonzero"
-                         {% if params.sentiment_method == 'nonzero' %}
-                           checked
-                         {% endif %}
-                         name="sentiment_method">
-                  <span class="icon" data-tooltip="Nonzero">
-                    <i class="fa-solid fa-face-meh-blank"></i>
-                  </span>
-                </label>
-              </div>
-            </div>
-
-            <label class="checkbox">
-              <input type="checkbox"
-                     name="check_sentiment"
-                     {% if params.check_sentiment == "on" %}
-                       checked
-                     {% endif %}
-                     data-script="on click toggle @disabled on #sliderWithValue then toggle @disabled on #sentiment">
-                Check sentiment
-            </label>
-          </div>
-          <div class="column is-narrow">
-            <div id="date">
-              <div class="field">
-                <div class="control">
-                  <input type="date" name="dates" value="{{ params.date }}">
-                  <script>
-                    var options = {
-                      "type": "datetime",
-                      "isRange": true,
-                      "color": "info",
-                      "validateLabel": "Save",
-                      "dateFormat": "yyyy-MM-dd",
-                      "startDate": "{{ params.from_date|escapejs }}",
-                      "startTime": "{{ params.from_time|escapejs }}",
-                      "endDate": "{{ params.to_date|escapejs }}",
-                      "endTime": "{{ params.to_time|escapejs }}",
-                    };
-                    // Initialize all input of type date
-                    var calendars = bulmaCalendar.attach('[type="date"]', options);
-
-                    // Loop on each calendar initialized
-                    for(var i = 0; i < calendars.length; i++) {
-                      // Add listener to select event
-                      calendars[i].on('save', date => {
-                        htmx.trigger("#search", "click");
-                      });
-                    }
-                  </script>
-                </div>
-              </div>
-              <div class="control">
-                <label class="radio button has-text-link">
-                  <input type="radio" value="desc" name="sorting"
-                         {% if params.sorting == None %}
-                           checked
-                         {% elif params.sorting == 'desc' %}
-                           checked
-                         {% endif %}
-                  >
-                  <span class="icon" data-tooltip="Sort descending">
-                    <i class="fa-solid fa-sort-down"></i>
-                  </span>
-                </label>
-                <label class="radio button">
-                  <input type="radio" value="asc" name="sorting"
-                         {% if params.sorting == 'asc' %}
-                           checked
-                         {% endif %}>
-                  <span class="icon" data-tooltip="Sort ascending">
-                    <i class="fa-solid fa-sort-up"></i>
-                  </span>
-                </label>
-                <label class="radio button">
-                  <input type="radio" value="none" name="sorting"
-                         {% if params.sorting == 'none' %}
-                           checked
-                         {% endif %}>
-                  <span class="icon" data-tooltip="No sort">
-                    <i class="fa-solid fa-sort"></i>
-                  </span>
-                </label>
-              </div>
-            </div>
-          </div>
-        </div>
-      </div>
-      <div class="is-hidden"></div>
-    </form>
-  </div>
-  <div class="block">
-    <input id="tags" class="input" type="tags" placeholder="Add query" value="{{ tags|joinsep:',' }}">
-    <script>
-      setupTags();
-    </script>
-  </div>
-  <div class="block">
-    <div id="results">
-      {% if results %}
-        {% include 'ui/drilldown/results.html' %}
-      {% endif %}
     </div>
   </div>
+
+  <script>
+    var grid = GridStack.init({
+      cellHeight: 20,
+      cellWidth: 50,
+      cellHeightUnit: 'px',
+      auto: true,
+      float: true,
+      draggable: {handle: '.panel-heading', scroll: false, appendTo: 'body'},
+      removable: false,
+      animate: true,
+    });
+    // GridStack.init();
+    setupTags();
+
+    // a widget is ready to be loaded
+    document.addEventListener('load-widget', function(event) {
+      let container = htmx.find('#drilldown-widget');
+      // get the scripts, they won't be run on the new element so we need to eval them
+      var scripts = htmx.findAll(container, "script");
+      let widgetelement = container.firstElementChild.cloneNode(true);
+
+      // check if there's an existing element like the one we want to swap
+      let grid_element = htmx.find('#grid-stack-main');
+      let existing_widget = htmx.find(grid_element, '#drilldown-widget-results');
+
+      // get the size and position attributes
+      if (existing_widget) {
+        let attrs = existing_widget.getAttributeNames();
+        for (let i = 0, len = attrs.length; i < len; i++) {
+          if (attrs[i].startsWith('gs-')) { // only target gridstack attributes
+            widgetelement.setAttribute(attrs[i], existing_widget.getAttribute(attrs[i]));
+          }
+        }
+      }
+
+      // clear the queue element
+      container.outerHTML = "";
+
+      // temporary workaround, other widgets can be duplicated, but not results
+      if (widgetelement.id == 'drilldown-widget-results') {
+        grid.removeWidget("drilldown-widget-{{ unique }}");
+      }
+
+      grid.addWidget(widgetelement);
+      // re-create the HTMX JS listeners, otherwise HTMX won't work inside the grid
+      htmx.process(widgetelement);
+
+      // update size when the widget is loaded
+      document.addEventListener('load-widget-results', function(evt) {
+        var added_widget = htmx.find(grid_element, '#drilldown-widget-results');
+        console.log(added_widget);
+        var itemContent = htmx.find(added_widget, ".control");
+        console.log(itemContent);
+        var scrollheight = itemContent.scrollHeight+80;
+        var verticalmargin = 0;
+        var cellheight = grid.opts.cellHeight;
+        var height = Math.ceil((scrollheight + verticalmargin) / (cellheight + verticalmargin));
+        var opts = {
+          h: height,
+        }
+        grid.update(
+          added_widget,
+          opts
+        );
+      });
+
+      // run the JS scripts inside the added element again
+      // for instance, this will fix the dropdown
+      for (var i = 0; i < scripts.length; i++) {
+        eval(scripts[i].innerHTML);
+      }
+    });
+  </script>
+
   <div id="modals-here">
   </div>
+  <div id="items-here">
+  </div>
+  <div id="widgets-here" style="display: none;">
+  </div>
+  <div id="results" style="display: none;">
+    {% if table %}
+      {% include 'widgets/table_results.html' %}
+    {% endif %}
+  </div>
+  <script>
+
+  </script>
+
 {% endblock %}

core/templates/ui/drilldown/results.html

@@ -1,212 +0,0 @@
-{% load static %}
-{% load index %}
-{% load joinsep %}
-{% include 'partials/notify.html' %}
-{% if results %}
-  <div style="display: none" id="jsonData" data-json="{{ data }}">
-  </div>
-  <div class="has-text-grey-light nowrap-parent">
-    <div class="nowrap-child block">
-      <i class="fa-solid fa-chart-mixed"></i>
-    </div>
-    <div class="nowrap-child">
-      <p>fetched {{ results|length }} of {{ card }} hits in {{ took }}ms</p>
-    </div>
-    {% if exemption is not None %}
-      <div class="nowrap-child">
-        <i class="fa-solid fa-book-bible"></i>
-      </div>
-    {% else %}
-      {% if redacted != 0 %}
-        <div class="nowrap-child">
-          <p>{{ redacted }} redacted</p>
-        </div>
-      {% endif %}
-    {% endif %}
-  </div>
-  <div class="box">
-    <div style="height: 30rem">
-      <canvas id="volume"></canvas>
-    </div>
-    <script src="{% static 'chart.js' %}"></script>
-  </div>
-  <div class="box">
-    <div class="table-container">
-      <table class="table is-striped is-hoverable is-fullwidth">
-        <thead>
-          <tr>
-            <th>src</th>
-            <th>type</th>
-            <th>ts</th>
-            <th>msg</th>
-            <th>host</th>
-            <th>nick</th>
-            <th>actions</th>
-            <th>channel</th>
-            <th>net</th>
-          </tr>
-        </thead>
-        <tbody>
-          {% for item in results %}
-            {# Workaround for curlylint #}
-            <tr class="{% if item.exemption == True %}
-                         has-background-grey-lighter
-                       {% elif item.type == 'join' %}
-                         has-background-success-light
-                       {% elif item.type == 'quit' %}
-                         has-background-danger-light
-                       {% elif item.type == 'kick' %}
-                         has-background-danger-light
-                       {% elif item.type == 'part' %}
-                         has-background-warning-light
-                       {% elif item.type == 'mode' %}
-                         has-background-info-light
-                       {% endif %}">
-              <td>
-                <a class="has-text-link is-underlined"
-                   onclick="populateSearch('src', '{{ item.src|escapejs }}')">
-                  {% if item.src == 'irc' %}
-                    <span class="icon" data-tooltip="IRC">
-                      <i class="fa-solid fa-hashtag" aria-hidden="true"></i>
-                    </span>
-                  {% elif item.src == 'dis' %}
-                    <span class="icon" data-tooltip="Discord">
-                      <i class="fa-brands fa-discord" aria-hidden="true"></i>
-                    </span>
-                  {% endif %}
-                </a>
-              </td>
-              <td>
-                <a class="has-text-link is-underlined"
-                   onclick="populateSearch('type', '{{ item.type|escapejs }}')">
-                  {% if item.type == 'msg' %}
-                    <span class="icon" data-tooltip="Message">
-                      <i class="fa-solid fa-message"></i>
-                    </span>
-                  {% elif item.type == 'join' %}
-                    <span class="icon" data-tooltip="Join">
-                      <i class="fa-solid fa-person-to-portal"></i>
-                    </span>
-                  {% elif item.type == 'part' %}
-                    <span class="icon" data-tooltip="Part">
-                      <i class="fa-solid fa-person-from-portal"></i>
-                    </span>
-                  {% elif item.type == 'quit' %}
-                    <span class="icon" data-tooltip="Quit">
-                      <i class="fa-solid fa-circle-xmark"></i>
-                    </span>
-                  {% elif item.type == 'kick' %}
-                    <span class="icon" data-tooltip="Kick">
-                      <i class="fa-solid fa-user-slash"></i>
-                    </span>
-                  {% elif item.type == 'nick' %}
-                    <span class="icon" data-tooltip="Nick">
-                      <i class="fa-solid fa-signature"></i>
-                    </span>
-                  {% elif item.type == 'mode' %}
-                    <span class="icon" data-tooltip="Mode">
-                      <i class="fa-solid fa-gear"></i>
-                    </span>
-                  {% elif item.type == 'action' %}
-                    <span class="icon" data-tooltip="Action">
-                      <i class="fa-solid fa-exclamation"></i>
-                    </span>
-                  {% else %}
-                    {{ item.type }}
-                  {% endif %}
-                </a>
-              </td>
-              <td>
-                <p>{{ item.date }}</p>
-                <p>{{ item.time }}</p>
-              </td>
-              <td style="max-width: 10em" class="wrap">{{ item.msg }}</td>
-              <td>
-                <a class="has-text-link is-underlined"
-                   onclick="populateSearch('host', '{{ item.host|escapejs }}')">
-                  {{ item.host }}
-                </a>
-              </td>
-              <td>
-                <div class="nowrap-parent">
-                  <div class="nowrap-child">
-                    {% if item.online is True %}
-                      <span class="icon has-text-success has-tooltip-success" data-tooltip="Online">
-                        <i class="fa-solid fa-circle"></i>
-                      </span>
-                    {% elif item.online is False %}
-                      <span class="icon has-text-danger has-tooltip-danger" data-tooltip="Offline">
-                        <i class="fa-solid fa-circle"></i>
-                      </span>
-                    {% else %}
-                      <span class="icon has-text-warning has-tooltip-warning" data-tooltip="Unknown">
-                        <i class="fa-solid fa-circle"></i>
-                      </span>
-                    {% endif %}
-                  </div>
-                  <a class="nowrap-child has-text-link is-underlined" onclick="populateSearch('nick', '{{ item.nick|escapejs }}')">
-                    {{ item.nick }}
-                  </a>
-                  {% if item.num_chans is not None %}
-                    <div class="nowrap-child">
-                      <span class="tag">
-                        {{ item.num_chans }}
-                      </span>
-                    </div>
-                  {% endif %}
-                </div>
-              </td>
-              <td>
-                {% if item.src == 'irc' %}
-                  <button
-                    hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
-                    hx-post="{% url 'modal_drilldown' %}"
-                    hx-vals='{"net": "{{ item.net }}", "nick": "{{ item.nick }}", "channel": "{{ item.channel }}"}'
-                    hx-target="#modals-here" 
-                    hx-trigger="click"
-                    class="button is-small">
-                    Information
-                  </button>
-                {% endif %}
-              </td>
-              <td>
-                <div class="nowrap-parent">
-                  <a class="nowrap-child has-text-link is-underlined"
-                     onclick="populateSearch('channel', '{{ item.channel|escapejs }}')">
-                    {{ item.channel }}
-                  </a>
-                  {% if item.num_users is not None %}
-                    <div class="nowrap-child">
-                      <span class="tag">
-                        {{ item.num_users }}
-                      </span>
-                    </div>
-                  {% endif %}
-                </div>
-              </td>
-              <td>
-                <a class="has-text-link is-underlined"
-                   onclick="populateSearch('net', '{{ item.net|escapejs }}')">
-                  {{ item.net }}
-                </a>
-              </td>
-            </tr>
-          {% endfor %}
-        </tbody>
-      </table>
-    </div>
-  </div>
-{% endif %}
-{# Update the tags in case the user changed the query #}
-{# Check for focus and refocus #}
-<script>
-  var inputTags = document.getElementsByClassName('tags-input');
-  var inputBox = document.querySelector("[placeholder='Add query']");
-  var isFocused = (document.activeElement === inputBox);
-  inputTags[0].outerHTML = '<input id="tags" class="input" type="tags" placeholder="Add query" value="{{ tags|joinsep:',' }}">';
-  setupTags();
-  var inputBox = document.querySelector("[placeholder='Add query']");
-  if (isFocused) {
-    inputBox.focus();
-  }
-</script>

core/templates/ui/drilldown/search_partial.html

@@ -0,0 +1,407 @@
+<form class="skipEmptyFields" method="POST" hx-post="{% url 'search' %}"
+  hx-trigger="change"
+  hx-target="#results"
+  hx-swap="innerHTML"
+  hx-indicator="#spinner">
+  {% csrf_token %}
+  <div class="columns">
+    <div class="column">
+      <div class="field has-addons">
+        <div id="query" class="control is-expanded has-icons-left">
+          <input
+            hx-post="{% url 'search' %}"
+            hx-trigger="keyup changed delay:200ms"
+            hx-target="#results"
+            hx-swap="innerHTML"
+            name="query"
+            value="{{ params.query }}"
+            class="input"
+            type="text"
+            placeholder="Search something">
+          <span class="icon is-small is-left">
+            <i class="fas fa-magnifying-glass"></i>
+          </span>
+        </div>
+        <div class="control">
+          <div class="field">
+            <button
+              id="search"
+              class="button is-info is-fullwidth"
+              hx-post="{% url 'search' %}"
+              hx-trigger="click"
+              hx-target="#results"
+              hx-swap="innerHTML">
+              Search
+            </button>
+          </div>
+        </div>
+      </div>
+    </div>
+    <div class="column is-3">
+      <div class="nowrap-parent">
+        <div
+          data-script="on click toggle .is-hidden on #options"
+          class="button is-light has-text-link is-right nowrap-child">
+          Options
+        </div>
+        <div class="nowrap-child">
+          <span id="spinner" class="button is-light has-text-link is-loading htmx-indicator">Static</span>
+        </div>
+      </div>
+    </div>
+  </div>
+  <div id="options" class="block is-hidden">
+    <div class="columns is-multiline">
+      <div class="column is-narrow">
+        <div class="field has-addons">
+          <div class="control has-icons-left">
+            <span class="select">
+              <select name="size">
+                {% for size in sizes %}
+                  {% if size == params.size %}
+                    <option selected value="{{ size }}">{{ size }}</option>
+                  {% else %}
+                    <option value="{{ size }}">{{ size }}</option>
+                  {% endif %}
+                {% endfor %}
+              </select>
+              <span class="icon is-small is-left">
+                <i class="fas fa-magnifying-glass"></i>
+              </span>
+            </span>
+          </div>
+          <p class="control">
+            <a class="button is-static">
+              results
+            </a>
+          </p>
+        </div>
+      </div>
+      <div class="column is-narrow">
+        <div class="field has-addons block">
+          <div class="control has-icons-left">
+            <span class="select">
+              <select id="source" name="source">
+                {% if params.source == 'irc' %}
+                  <option selected value="irc">IRC</option>
+                {% else %}
+                  <option value="irc">IRC</option>
+                {% endif %}
+
+                {% if params.source == 'dis' %}
+                  <option selected value="dis">Discord</option>
+                {% else %}
+                  <option value="dis">Discord</option>
+                {% endif %}
+
+                {% if params.source == None %}
+                  <option selected value="4ch">4chan</option>
+                {% elif params.source == '4ch' %}
+                  <option selected value="4ch">4chan</option>
+                {% else %}
+                  <option value="4ch">4chan</option>
+                {% endif %}
+
+                {% if params.source == 'all' %}
+                  <option selected value="all">All</option>
+                {% else %}
+                  <option value="all">All</option>
+                {% endif %}
+
+              </select>
+              <span class="icon is-small is-left">
+                <i class="fas fa-magnifying-glass"></i>
+              </span>
+            </span>
+          </div>
+          <p class="control">
+            <a class="button is-static">
+              source
+            </a>
+          </p>
+        </div>
+        <div class="control">
+          <div class="field">
+            <input
+              id="dedup_switch"
+              type="checkbox"
+              class="switch is-rounded is-info"
+              name="dedup"
+              {% if params.dedup == "on" %}
+                checked="checked"
+              {% endif %}>
+            <label
+              for="dedup_switch">
+              Deduplicate results
+            </label>
+          </div>
+        </div>
+      </div>
+      <div class="column is-narrow">
+        <div id="sentiment">
+          <div class="field has-addons">
+            <div class="control">
+              <input
+                {% if params.check_sentiment != "on" %}
+                  disabled="undefined"
+                {% endif %}
+                name="sentiment" id="sliderWithValue" class="slider has-output-tooltip is-fullwidth" min="-1" max="1"
+                {% if params.sentiment == None %}
+                  value="0"
+                {% else %}
+                  value="{{ params.sentiment }}"
+                {% endif %}
+                step="0.05" type="range">
+              <output for="sliderWithValue" class="slider-output">
+                {% if params.sentiment == None %}
+                  0
+                {% else %}
+                  {{ params.sentiment }}
+                {% endif %}
+              </output>
+              <script>bulmaSlider.attach();</script>
+            </div>
+            <p class="control">
+              <a class="button is-static">
+                sentiment
+              </a>
+            </p>
+          </div>
+          <div class="control">
+            <label class="radio button has-text-link">
+              <input
+                type="radio"
+                value="below"
+                class="sentiment-radio"
+                {% if params.sentiment_method == 'below' %}
+                  checked
+                {% endif %}
+                name="sentiment_method"
+                {% if params.check_sentiment != "on" %}
+                  disabled
+                {% endif %}>
+              <span class="icon" data-tooltip="Below">
+                <i class="fa-solid fa-face-frown"></i>
+              </span>
+            </label>
+            <label class="radio button has-text-link is-hidden">
+              <input
+                type="radio"
+                value="exact"
+                class="sentiment-radio"
+                {% if params.sentiment_method == 'exact' %}
+                  checked
+                {% endif %}
+                name="sentiment_method"
+                {% if params.check_sentiment != "on" %}
+                  disabled
+                {% endif %}>
+              <span class="icon" data-tooltip="Exact">
+                <i class="fa-solid fa-face-smile"></i>
+              </span>
+            </label>
+            <label class="radio button has-text-link">
+              <input
+                type="radio"
+                value="above"
+                class="sentiment-radio"
+                {% if params.sentiment_method == 'above' %}
+                  checked
+                {% endif %}
+                name="sentiment_method"
+                {% if params.check_sentiment != "on" %}
+                  disabled
+                {% endif %}>
+              <span class="icon" data-tooltip="Above">
+                <i class="fa-solid fa-face-smile"></i>
+              </span>
+            </label>
+            <label class="radio button has-text-link">
+              <input
+                type="radio"
+                value="nonzero"
+                class="sentiment-radio"
+                {% if params.sentiment_method == 'nonzero' %}
+                  checked
+                {% endif %}
+                name="sentiment_method"
+                {% if params.check_sentiment != "on" %}
+                  disabled
+                {% endif %}>
+              <span class="icon" data-tooltip="Nonzero">
+                <i class="fa-solid fa-face-meh-blank"></i>
+              </span>
+            </label>
+          </div>
+
+        </div>
+        <div class="control">
+          <div class="field">
+            <input
+              id="sentiment_switch"
+              type="checkbox"
+              class="switch is-rounded is-info"
+              name="check_sentiment"
+              data-script="on click toggle @disabled on #sliderWithValue then toggle @disabled on #sentiment then toggle @disabled on .sentiment-radio"
+              {% if params.check_sentiment == "on" %}
+                checked
+              {% endif %}>
+            <label
+              for="sentiment_switch">
+              Check sentiment
+            </label>
+          </div>
+        </div>
+        <div class="control">
+          <div class="field">
+            <input
+              id="sentiment_graph_switch"
+              type="checkbox"
+              class="switch is-rounded is-info"
+              name="show_sentiment"
+              data-script="on click toggle .is-hidden on #sentiment-container">
+
+            <label
+              for="sentiment_graph_switch">
+              Show graph
+            </label>
+          </div>
+        </div>
+      </div>
+      <div class="column is-narrow">
+        <div id="date">
+          <div class="field">
+            <div class="control">
+              <input type="date" name="dates" value="{{ params.date }}">
+              <script>
+                var options = {
+                  "type": "datetime",
+                  "isRange": true,
+                  "color": "info",
+                  "validateLabel": "Save",
+                  "dateFormat": "yyyy-MM-dd",
+                  "startDate": "{{ params.from_date|escapejs }}",
+                  "startTime": "{{ params.from_time|escapejs }}",
+                  "endDate": "{{ params.to_date|escapejs }}",
+                  "endTime": "{{ params.to_time|escapejs }}",
+                  "displayMode": "dialog"
+                };
+                // Initialize all input of type date
+                var calendars = bulmaCalendar.attach('[type="date"]', options);
+
+                // Loop on each calendar initialized
+                for(var i = 0; i < calendars.length; i++) {
+                  // Add listener to select event
+                  calendars[i].on('save', date => {
+                    htmx.trigger("#search", "click");
+                  });
+                }
+              </script>
+            </div>
+          </div>
+          <div class="control">
+            <label class="radio button has-text-link">
+              <input
+                type="radio"
+                value="desc"
+                name="sorting"
+                {% if params.sorting == None %}
+                  checked
+                {% elif params.sorting == 'desc' %}
+                  checked
+                {% endif %}>
+              <span class="icon" data-tooltip="Sort descending">
+                <i class="fa-solid fa-sort-down"></i>
+              </span>
+            </label>
+            <label class="radio button">
+              <input
+                type="radio"
+                value="asc"
+                name="sorting"
+                {% if params.sorting == 'asc' %}
+                  checked
+                {% endif %}>
+              <span class="icon" data-tooltip="Sort ascending">
+                <i class="fa-solid fa-sort-up"></i>
+              </span>
+            </label>
+            <label class="radio button">
+              <input
+                type="radio"
+                value="none"
+                name="sorting"
+                {% if params.sorting == 'none' %}
+                  checked
+                {% endif %}>
+              <span class="icon" data-tooltip="No sort">
+                <i class="fa-solid fa-sort"></i>
+              </span>
+            </label>
+          </div>
+        </div>
+      </div>
+      <div class="column is-narrow rounded-tooltip">
+        <div class="field has-addons">
+          <div class="control has-icons-left">
+            <span class="select is-warning">
+              <select {% if not user.is_superuser %}disabled{% endif %} id="index" name="index">
+                {% if params.index == 'main' %}
+                  <option selected value="main">Main</option>
+                {% elif params.index == None %}
+                  <option selected value="main">Main</option>
+                {% else %}
+                  <option value="main">Main</option>
+                {% endif %}
+
+                {% if params.index == 'internal' %}
+                  <option selected value="internal">Internal</option>
+                {% else %}
+                  <option value="internal">Internal</option>
+                {% endif %}
+
+                {% if params.index == 'meta' %}
+                  <option selected value="meta">Meta</option>
+                {% else %}
+                  <option value="meta">Meta</option>
+                {% endif %}
+
+                {% if params.index == 'restricted' %}
+                  <option selected value="restricted">Restricted</option>
+                {% else %}
+                  <option value="restricted">Restricted</option>
+                {% endif %}
+
+              </select>
+              <span class="icon is-small is-left">
+                <i class="fas fa-magnifying-glass"></i>
+              </span>
+            </span>
+          </div>
+          <p class="control">
+            <a class="button is-static">
+              index
+            </a>
+          </p>
+        </div>
+        {% if not user.is_superuser %}
+          <span class="tooltiptext tag is-danger is-light">No access</span>
+        {% endif %}
+      </div>
+    </div>
+  </div>
+  <div class="block">
+    <input
+      hx-trigger="change"
+      hx-post="{% url 'search' %}"
+      hx-target="#results"
+      hx-swap="innerHTML"
+      id="tags"
+      class="input"
+      type="tags"
+      name="tags"
+      placeholder="Tag search: nick: john"
+      value="{{ params.tags }}">
+  </div>
+  <div class="is-hidden"></div>
+</form>

core/templates/ui/drilldown/sentiment_partial.html

@@ -0,0 +1,10 @@
+{% load static %}
+<div style="display: none" id="jsonData" data-json="{{ data }}">
+</div>
+
+{% if params.index != 'int' and params.index != 'meta' %}
+  <div id="sentiment-container" {% if params.show_sentiment is None %} class="is-hidden" {% endif %}>
+    <canvas id="sentiment-chart"></canvas>
+  </div>
+  <script src="{% static 'chart.js' %}"></script>
+{% endif %}

core/templates/ui/drilldown/table_results.html

@@ -1,32 +0,0 @@
-{% load static %}
-{% load index %}
-{% include 'partials/notify.html' %}
-{% if table %}
-  <div style="display: none" id="jsonData" data-json="{{ data }}">
-  </div>
-
-  <div class="has-text-grey-light nowrap-parent">
-    <div class="nowrap-child block">
-      <i class="fa-solid fa-chart-mixed"></i>
-    </div>
-    <div class="nowrap-child">
-      <p>fetched {{ results|length }} of {{ card }} hits in {{ took }}ms</p>
-    </div>
-    {% if exemption is not None %}
-      <div class="nowrap-child">
-        <i class="fa-solid fa-book-bible"></i>
-      </div>
-    {% else %}
-      {% if redacted != 0 %}
-        <div class="nowrap-child">
-          <p>{{ redacted }} redacted</p>
-        </div>
-      {% endif %}
-    {% endif %}
-  </div>
-  <div class="box">
-    <div class="table-container">
-      {% include 'ui/drilldown/table_results_partial.html' %}
-    </div>
-  </div>
-{% endif %}

core/templates/ui/drilldown/table_results_partial.html

@@ -1,75 +1,488 @@
-{% extends 'django-tables2/bulma.html' %}
-
 {% load django_tables2 %}
-
-{% load i18n %}
-
-{% block table.thead %}
-  {% if table.show_header %}
-    <thead {{ table.attrs.thead.as_html }}>
-      <tr>
-        {% for column in table.columns %}
-          <th
-            {{ column.attrs.th.as_html }}
-            hx-post="{% querystring table.prefixed_order_by_field=column.order_by_alias.next %}"
-            hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
-            hx-trigger="click"
-            hx-target="div.table-container"
-            hx-swap="outerHTML"
-            hx-indicator=".progress"
-            style="cursor: pointer;">
-            {{ column.header }}
-          </th>
-        {% endfor %}
-      </tr>
-    </thead>
-  {% endif %}
-{% endblock table.thead %}
-
-{# Pagination block overrides #}
-{% block pagination.previous %}
-  <li class="previous page-item">
-    <div
-      hx-post="{% querystring table.prefixed_page_field=table.page.previous_page_number %}"
-      hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
-      hx-trigger="click"
-      hx-target="div.table-container"
-      hx-swap="outerHTML"
-      hx-indicator=".progress"
-      class="page-link">
-      <span aria-hidden="true">&laquo;</span>
-      {% trans 'previous' %}
-    </div>
-  </li>
-{% endblock pagination.previous %}
-{% block pagination.range %}
-  {% for p in table.page|table_page_range:table.paginator %}
-    <li class="page-item{% if table.page.number == p %} active{% endif %}">
-      <div
-        class="page-link" 
-        {% if p != '...' %}hx-post="{% querystring table.prefixed_page_field=p %}"{% endif %}
-        hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
-        hx-trigger="click"
-        hx-target="div.table-container"
-        hx-swap="outerHTML"
-        hx-indicator=".progress">
-        {{ p }}
+{% load django_tables2_bulma_template %}
+{% load static %}
+{% load joinsep %}
+{% load urlsafe %}
+{% block table-wrapper %}
+  <div id="drilldown-table" class="column-shifter-container" style="position:relative; z-index:1;">
+    {% block table %}
+      <div class="nowrap-parent">
+        <div class="nowrap-child">
+          <div class="dropdown" id="dropdown">
+            <div class="dropdown-trigger">
+              <button id="dropdown-trigger" class="button dropdown-toggle" aria-haspopup="true" aria-controls="dropdown-menu">
+                <span>Show/hide fields</span>
+                <span class="icon is-small">
+                  <i class="fas fa-angle-down" aria-hidden="true"></i>
+                </span>
+              </button>
+            </div>
+            <div class="dropdown-menu" id="dropdown-menu" role="menu">
+              <div class="dropdown-content" style="position:absolute; z-index:2;">
+                {% for column in table.columns %}
+                  {% if column.name in show %}
+                    <a class="btn-shift-column dropdown-item"
+                      data-td-class="{{ column.name }}"
+                      data-state="on"
+                      {% if not forloop.last %} style="border-bottom:1px solid #ccc;" {%endif %}
+                      data-table-class-container="drilldown-table">
+                      <span class="check icon" data-tooltip="Visible" style="display:none;">
+                        <i class="fa-solid fa-check"></i>
+                      </span>
+                      <span class="uncheck icon" data-tooltip="Hidden" style="display:none;">
+                        <i class="fa-solid fa-xmark"></i>
+                      </span>
+                      {{ column.header }}
+                    </a>
+                  {% endif %}
+                {% endfor %}
+              </div>
+            </div>
+          </div>
+        </div>
+        <div class="nowrap-child">
+          <span id="loader" class="button is-light has-text-link is-loading">Static</span>
+        </div>
       </div>
-    </li>
-  {% endfor %}
-{% endblock pagination.range %}
-{% block pagination.next %}
-  <li class="next page-item">
-    <div
-      hx-post="{% querystring table.prefixed_page_field=table.page.next_page_number %}"
-      hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
-      hx-trigger="click"
-      hx-target="div.table-container"
-      hx-swap="outerHTML"
-      hx-indicator=".progress"
-      class="page-link">
-      {% trans 'next' %}
-      <span aria-hidden="true">&raquo;</span>
-    </div>
-  </li>
-{% endblock pagination.next %}
+      <script>
+        var dropdown_button = document.getElementById("dropdown-trigger");
+        var dropdown = document.getElementById("dropdown");
+        dropdown_button.addEventListener('click', function(e) {
+          // elements[i].preventDefault();
+          dropdown.classList.toggle('is-active');
+        });
+
+      </script>
+      <div id="table-container" style="display:none;">
+        <table {% render_attrs table.attrs class="table drilldown-results-table is-fullwidth" %}>
+          {% block table.thead %}
+            {% if table.show_header %}
+              <thead {% render_attrs table.attrs.thead class="" %}>
+                {% block table.thead.row %}
+                  <tr>
+                    {% for column in table.columns %}
+                      {% if column.name in show %}
+                        {% block table.thead.th %}
+                          <th class="orderable {{ column.name }}">
+                            <div class="nowrap-parent">
+                              {% if column.orderable %}
+                                <div class="nowrap-child">
+                                  {% if column.is_ordered %}
+                                    {% is_descending column.order_by as descending %}
+                                    {% if descending %}
+                                      <span class="icon" aria-hidden="true">{% block table.desc_icon %}<i class="fa-solid fa-sort-down"></i>{% endblock table.desc_icon %}</span>
+                                    {% else %}
+                                      <span class="icon" aria-hidden="true">{% block table.asc_icon %}<i class="fa-solid fa-sort-up"></i>{% endblock table.asc_icon %}</span>
+                                    {% endif %}
+                                  {% else %}
+                                    <span class="icon" aria-hidden="true">{% block table.orderable_icon %}<i class="fa-solid fa-sort"></i>{% endblock table.orderable_icon %}</span>
+                                  {% endif %}
+                                </div>
+                                <div class="nowrap-child">
+                                  <a
+                                    hx-get="search/{% querystring table.prefixed_order_by_field=column.order_by_alias.next %}&{{ uri }}"
+                                    hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
+                                    hx-trigger="click"
+                                    hx-target="#results"
+                                    hx-swap="innerHTML"
+                                    hx-indicator="#spinner"
+                                    style="cursor: pointer;">
+                                    {{ column.header }}
+                                  </a>
+                                </div>
+                              {% else %}
+                                <div class="nowrap-child">
+                                  {{ column.header }}
+                                </div>
+                              {% endif %}
+                            </div>
+                          </th>
+                        {% endblock table.thead.th %}
+                      {% endif %}
+
+                    {% endfor %}
+                  </tr>
+                {% endblock table.thead.row %}
+              </thead>
+            {% endif %}
+          {% endblock table.thead %}
+          {% block table.tbody %}
+            <tbody {{ table.attrs.tbody.as_html }}>
+              {% for row in table.paginated_rows %}
+                {% block table.tbody.row %}
+                  {% if row.cells.type == 'control' %}
+                    <tr>
+                      <td></td>
+                      <td>
+                        <span class="icon has-text-grey" data-tooltip="Hidden">
+                          <i class="fa-solid fa-file-slash"></i>
+                        </span>
+                      </td>
+                      <td>
+                        <p class="has-text-grey">Hidden {{ row.cells.hidden }} similar result{% if row.cells.hidden > 1%}s{% endif %}</p>
+                      </td>
+
+                    </tr>
+                  {% else %}
+                    <tr class="
+                      {% if row.cells.exemption == True %}has-background-grey-lighter
+                      {% elif cell == 'join' %}has-background-success-light
+                      {% elif cell == 'quit' %}has-background-danger-light
+                      {% elif cell == 'kick' %}has-background-danger-light
+                      {% elif cell == 'part' %}has-background-warning-light
+                      {% elif cell == 'mode' %}has-background-info-light
+                      {% endif %}">
+                      {% for column, cell in row.items %}
+                        {% if column.name in show %}
+                          {% block table.tbody.td %}
+                            {% if cell == '—' %}
+                              <td class="{{ column.name }}">
+                                <span class="icon">
+                                  <i class="fa-solid fa-file-slash"></i>
+                                </span>
+                              </td>
+                            {% elif column.name == 'tokens' %}
+                              <td class="{{ column.name }} wrap" style="max-width: 10em">
+                                {{ cell|joinsep:',' }}
+                              </td>
+                            {% elif column.name == 'src' %}
+                              <td class="{{ column.name }}">
+                                <a
+                                  class="has-text-link is-underlined"
+                                  onclick="populateSearch('src', '{{ cell|escapejs }}')">
+                                  {% if row.cells.src == 'irc' %}
+                                    <span class="icon" data-tooltip="IRC">
+                                      <i class="fa-solid fa-hashtag" aria-hidden="true"></i>
+                                    </span>
+                                  {% elif row.cells.src == 'dis' %}
+                                    <span class="icon" data-tooltip="Discord">
+                                      <i class="fa-brands fa-discord" aria-hidden="true"></i>
+                                    </span>
+                                  {% elif row.cells.src == '4ch' %}
+                                    <span class="icon" data-tooltip="4chan">
+                                      <i class="fa-solid fa-leaf" aria-hidden="true"></i>
+                                    </span>
+                                  {% endif %}
+                                </a>
+                              </td>
+                            {% elif column.name == 'ts' %}
+                              <td class="{{ column.name }}">
+                                <p>{{ row.cells.date }}</p>
+                                <p>{{ row.cells.time }}</p>
+                              </td>
+                            {% elif column.name == 'type' or column.name == 'mtype' %}
+                              <td class="{{ column.name }}">
+                                <a
+                                  class="has-text-link is-underlined"
+                                  onclick="populateSearch('{{ column.name }}', '{{ cell|escapejs }}')">
+                                  {% if cell == 'msg' %}
+                                    <span class="icon" data-tooltip="Message">
+                                      <i class="fa-solid fa-message"></i>
+                                    </span>
+                                  {% elif cell == 'join' %}
+                                    <span class="icon" data-tooltip="Join">
+                                      <i class="fa-solid fa-person-to-portal"></i>
+                                    </span>
+                                  {% elif cell == 'part' %}
+                                    <span class="icon" data-tooltip="Part">
+                                      <i class="fa-solid fa-person-from-portal"></i>
+                                    </span>
+                                  {% elif cell == 'quit' %}
+                                    <span class="icon" data-tooltip="Quit">
+                                      <i class="fa-solid fa-circle-xmark"></i>
+                                    </span>
+                                  {% elif cell == 'kick' %}
+                                    <span class="icon" data-tooltip="Kick">
+                                      <i class="fa-solid fa-user-slash"></i>
+                                    </span>
+                                  {% elif cell == 'nick' %}
+                                    <span class="icon" data-tooltip="Nick">
+                                      <i class="fa-solid fa-signature"></i>
+                                    </span>
+                                  {% elif cell == 'mode' %}
+                                    <span class="icon" data-tooltip="Mode">
+                                      <i class="fa-solid fa-gear"></i>
+                                    </span>
+                                  {% elif cell == 'action' %}
+                                    <span class="icon" data-tooltip="Action">
+                                      <i class="fa-solid fa-exclamation"></i>
+                                    </span>
+                                  {% elif cell == 'notice' %}
+                                    <span class="icon" data-tooltip="Notice">
+                                      <i class="fa-solid fa-message-code"></i>
+                                    </span>
+                                  {% elif cell == 'conn' %}
+                                    <span class="icon" data-tooltip="Connection">
+                                      <i class="fa-solid fa-cloud-exclamation"></i>
+                                    </span>
+                                  {% elif cell == 'znc' %}
+                                    <span class="icon" data-tooltip="ZNC">
+                                      <i class="fa-brands fa-unity"></i>
+                                    </span>
+                                  {% elif cell == 'query' %}
+                                    <span class="icon" data-tooltip="Query">
+                                      <i class="fa-solid fa-message"></i>
+                                    </span>
+                                  {% elif cell == 'highlight' %}
+                                    <span class="icon" data-tooltip="Highlight">
+                                      <i class="fa-solid fa-exclamation"></i>
+                                    </span>
+                                  {% elif cell == 'who' %}
+                                    <span class="icon" data-tooltip="Who">
+                                      <i class="fa-solid fa-passport"></i>
+                                    </span>
+                                  {% elif cell == 'topic' %}
+                                    <span class="icon" data-tooltip="Topic">
+                                      <i class="fa-solid fa-sign"></i>
+                                    </span>
+                                  {% else %}
+                                    {{ cell }}
+                                  {% endif %}
+                                </a>
+                              </td>
+                            {% elif column.name == 'msg' %}
+                              <td class="{{ column.name }} wrap">
+                                <a
+                                  class="has-text-grey is-underlined"
+                                  hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
+                                  hx-post="{% url 'modal_context' %}"
+                                  hx-vals='{"net": "{{ row.cells.net|escapejs }}",
+                                  "num": "{{ row.cells.num|escapejs }}",
+                                  "source": "{{ row.cells.src|escapejs }}",
+                                  "channel": "{{ row.cells.channel|escapejs }}",
+                                  "time": "{{ row.cells.time|escapejs }}",
+                                  "date": "{{ row.cells.date|escapejs }}",
+                                  "index": "{{ params.index }}",
+                                  "type": "{{ row.cells.type }}",
+                                  "mtype": "{{ row.cells.mtype }}",
+                                  "nick": "{{ row.cells.nick|escapejs }}",
+                                  "dedup": "{{ params.dedup }}"}'
+                                  hx-target="#modals-here"
+                                  hx-trigger="click"
+                                  href="/?modal=context&net={{row.cells.net|escapejs}}&num={{row.cells.num|escapejs}}&source={{row.cells.src|escapejs}}&channel={{row.cells.channel|urlsafe}}&time={{row.cells.time|escapejs}}&date={{row.cells.date|escapejs}}&index={{params.index}}&type={{row.cells.type}}&mtype={{row.cells.mtype}}&nick={{row.cells.mtype|escapejs}}">
+                                  {{ row.cells.msg }}
+                                </a>
+                              </td>
+                            {% elif column.name == 'nick' %}
+                              <td class="{{ column.name }}">
+                                <div class="nowrap-parent">
+                                  <div class="nowrap-child">
+                                    {% if row.cells.online is True %}
+                                      <span class="icon has-text-success has-tooltip-success" data-tooltip="Online">
+                                        <i class="fa-solid fa-circle"></i>
+                                      </span>
+                                    {% elif row.cells.online is False %}
+                                      <span class="icon has-text-danger has-tooltip-danger" data-tooltip="Offline">
+                                        <i class="fa-solid fa-circle"></i>
+                                      </span>
+                                    {% else %}
+                                      <span class="icon has-text-warning has-tooltip-warning" data-tooltip="Unknown">
+                                        <i class="fa-solid fa-circle"></i>
+                                      </span>
+                                    {% endif %}
+                                  </div>
+                                  <a class="nowrap-child has-text-link is-underlined" onclick="populateSearch('nick', '{{ cell|escapejs }}')">
+                                    {{ cell }}
+                                  </a>
+                                  <div class="nowrap-child">
+                                    {% if row.cells.src == 'irc' %}
+                                      <a
+                                        hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
+                                        hx-post="{% url 'modal_drilldown' %}"
+                                        hx-vals='{"net": "{{ row.cells.net }}", "nick": "{{ row.cells.nick }}", "channel": "{{ row.cells.channel }}"}'
+                                        hx-target="#modals-here"
+                                        hx-trigger="click"
+                                        class="has-text-black">
+                                        <span class="icon" data-tooltip="Open drilldown modal">
+                                          <i class="fa-solid fa-album"></i>
+                                        </span>
+                                      </a>
+                                      <a
+                                        hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
+                                        hx-post="{% url 'modal_drilldown' type='window' %}"
+                                        hx-vals='{"net": "{{ row.cells.net }}", "nick": "{{ row.cells.nick }}", "channel": "{{ row.cells.channel }}"}'
+                                        hx-target="#items-here"
+                                        hx-swap="afterend"
+                                        hx-trigger="click"
+                                        class="has-text-black">
+                                        <span class="icon" data-tooltip="Open drilldown window">
+                                          <i class="fa-solid fa-album"></i>
+                                        </span>
+                                      </a>
+                                      <a
+                                        hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
+                                        hx-post="{% url 'modal_drilldown' type='widget' %}"
+                                        hx-vals='{"net": "{{ row.cells.net }}", "nick": "{{ row.cells.nick }}", "channel": "{{ row.cells.channel }}"}'
+                                        hx-target="#widgets-here"
+                                        hx-trigger="click"
+                                        class="has-text-black">
+                                        <span class="icon" data-tooltip="Open drilldown widget">
+                                          <i class="fa-solid fa-album"></i>
+                                        </span>
+                                      </a>
+                                    {% endif %}
+                                  </div>
+                                  {% if row.cells.num_chans != '—' %}
+                                    <div class="nowrap-child">
+                                      <span class="tag">
+                                        {{ row.cells.num_chans }}
+                                      </span>
+                                    </div>
+                                  {% endif %}
+                                </div>
+                              </td>
+                            {% elif column.name == 'channel' %}
+                              <td class="{{ column.name }}">
+                                {% if cell != '—' %}
+                                  <div class="nowrap-parent">
+                                    <a
+                                      class="nowrap-child has-text-link is-underlined"
+                                      onclick="populateSearch('channel', '{{ cell|escapejs }}')">
+                                      {{ cell }}
+                                    </a>
+                                    {% if row.cells.num_users != '—' %}
+                                      <div class="nowrap-child">
+                                        <span class="tag">
+                                          {{ row.cells.num_users }}
+                                        </span>
+                                      </div>
+                                    {% endif %}
+                                  </div>
+                                {% else %}
+                                  {{ cell }}
+                                {% endif %}
+                              </td>
+                            {% elif cell is True or cell is False %}
+                              <td class="{{ column.name }}">
+                                {% if cell is True %}
+                                  <span class="icon has-text-success">
+                                    <i class="fa-solid fa-check"></i>
+                                  </span>
+                                {% else %}
+                                  <span class="icon">
+                                    <i class="fa-solid fa-xmark"></i>
+                                  </span>
+                                {% endif %}
+                              </td>
+                            {% else %}
+                              <td class="{{ column.name }}">
+                                <a
+                                  class="has-text-link is-underlined"
+                                  onclick="populateSearch('{{ column.name }}', '{{ cell|escapejs }}')">
+                                  {{ cell }}
+                                </a>
+                              </td>
+                            {% endif %}
+                          {% endblock table.tbody.td %}
+                        {% endif %}
+                      {% endfor %}
+                    </tr>
+                  {% endif %}
+                {% endblock table.tbody.row %}
+              {% empty %}
+                {% if table.empty_text %}
+                  {% block table.tbody.empty_text %}
+                    <tr><td class="{{ column.name }}" colspan="{{ table.columns|length }}">{{ table.empty_text }}</td></tr>
+                  {% endblock table.tbody.empty_text %}
+                {% endif %}
+              {% endfor %}
+            </tbody>
+          {% endblock table.tbody %}
+          {% block table.tfoot %}
+            {% if table.has_footer %}
+              <tfoot {{ table.attrs.tfoot.as_html }}>
+                {% block table.tfoot.row %}
+                  <tr>
+                    {% for column in table.columns %}
+                      {% block table.tfoot.td %}
+                        <td class="{{ column.name }}" {{ column.attrs.tf.as_html }}>{{ column.footer }}</td>
+                      {% endblock table.tfoot.td %}
+                    {% endfor %}
+                  </tr>
+                {% endblock table.tfoot.row %}
+              </tfoot>
+            {% endif %}
+          {% endblock table.tfoot %}
+        </table>
+      </div>
+    {% endblock table %}
+    {% block pagination %}
+      {% if table.page and table.paginator.num_pages > 1 %}
+        <nav class="pagination is-justify-content-flex-end" role="navigation" aria-label="pagination">
+          {% block pagination.previous %}
+            <a
+              class="pagination-previous is-flex-grow-0 {% if not table.page.has_previous %}is-hidden-mobile{% endif %}"
+              {% if table.page.has_previous %}
+                hx-get="search/{% querystring table.prefixed_page_field=table.page.previous_page_number %}&{{ uri }}"
+                hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
+                hx-trigger="click"
+                hx-target="#results"
+                hx-swap="innerHTML"
+                hx-indicator="#spinner"
+              {% else %}
+                href="#"
+                disabled
+              {% endif %}
+              style="order:1;">
+              {% block pagination.previous.text %}
+                <span aria-hidden="true">&laquo;</span>
+              {% endblock pagination.previous.text  %}
+            </a>
+          {% endblock pagination.previous %}
+          {% block pagination.next %}
+            <a
+              class="pagination-next is-flex-grow-0 {% if not table.page.has_next %}is-hidden-mobile{% endif %}"
+              {% if table.page.has_next %}
+                hx-get="search/{% querystring table.prefixed_page_field=table.page.next_page_number %}&{{ uri }}"
+                hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
+                hx-trigger="click"
+                hx-target="#results"
+                hx-swap="innerHTML"
+                hx-indicator="#spinner"
+              {% else %}
+                href="#"
+                disabled
+              {% endif %}
+              style="order:3;"
+            >
+              {% block pagination.next.text %}
+                <span aria-hidden="true">&raquo;</span>
+              {% endblock pagination.next.text  %}
+            </a>
+          {% endblock pagination.next %}
+          {% if table.page.has_previous or table.page.has_next %}
+            {% block pagination.range %}
+              <ul class="pagination-list is-flex-grow-0" style="order:2;">
+                {% for p in table.page|table_page_range:table.paginator %}
+                  <li>
+                    <a
+                      class="pagination-link {% if p == table.page.number %}is-current{% endif %}"
+                      aria-label="Page {{ p }}" block
+                      {% if p == table.page.number %}aria-current="page"{% endif %}
+                      {% if p == table.page.number %}
+                        href="#"
+                      {% else %}
+                        hx-get="search/{% querystring table.prefixed_page_field=p %}&{{ uri }}"
+                        hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
+                        hx-trigger="click"
+                        hx-target="#results"
+                        hx-swap="innerHTML"
+                        hx-indicator="#spinner"
+                      {% endif %}
+                    >
+                      {% if p == '...' %}
+                        <span class="pagination-ellipsis">&hellip;</span>
+                      {% else %}
+                        {{ p }}
+                      {% endif %}
+                    </a>
+                  </li>
+                {% endfor %}
+              </ul>
+            {% endblock pagination.range %}
+          {% endif %}
+        </nav>
+      {% endif %}
+    {% endblock pagination %}
+  </div>
+{% endblock table-wrapper %}

core/templates/ui/insights/channels.html

@@ -11,7 +11,7 @@
                 hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
                 hx-post="{% url 'modal_drilldown' %}"
                 hx-vals='{"net": "{{ net }}", "nick": "{{ nick }}", "channel": "{{ chan }}"}'
-                hx-target="#modals-here" 
+                hx-target="#modals-here"
                 hx-trigger="click"
                 class="button is-small">
                 {{ chan }}

core/templates/ui/insights/info.html

@@ -19,6 +19,7 @@
   hx-swap="outerHTML">
 </div>
 <div id="info">
+  {% include 'partials/notify.html' %}
   {% if item is not None %}
     <div class="content" style="max-height: 30em; overflow: auto;">
       <div class="table-container">
@@ -82,7 +83,7 @@
                     hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
                     hx-post="{% url 'modal_insights' %}"
                     hx-vals='{"net": "{{ item.net }}", "nick": "{{ item.nick }}", "channel": "{{ item.channel }}"}'
-                    hx-target="#modals-here" 
+                    hx-target="#modals-here"
                     hx-trigger="click"
                     class="button is-small">
                     Information

core/templates/ui/insights/insights.html

@@ -1,38 +1,39 @@
 {% extends "base.html" %}
 {% load static %}
 {% block content %}
+  {% include 'partials/notify.html' %}
   <script>
     // tabbed browsing for the modal
     function initTabs() {
-    TABS.forEach((tab) => {
-      tab.addEventListener('click', (e) => {
-        let selected = tab.getAttribute('data-tab');
-        updateActiveTab(tab);
-        updateActiveContent(selected);
+      TABS.forEach((tab) => {
+        tab.addEventListener('click', (e) => {
+          let selected = tab.getAttribute('data-tab');
+          updateActiveTab(tab);
+          updateActiveContent(selected);
+        })
       })
-    })
-  }
+    }
 
-  function updateActiveTab(selected) {
-    TABS.forEach((tab) => {
-      if (tab && tab.classList.contains(ACTIVE_CLASS)) {
-        tab.classList.remove(ACTIVE_CLASS);
-      }
-    });
-    selected.classList.add(ACTIVE_CLASS);
-  }
+    function updateActiveTab(selected) {
+      TABS.forEach((tab) => {
+        if (tab && tab.classList.contains(ACTIVE_CLASS)) {
+          tab.classList.remove(ACTIVE_CLASS);
+        }
+      });
+      selected.classList.add(ACTIVE_CLASS);
+    }
 
-  function updateActiveContent(selected) {
-    CONTENT.forEach((item) => {
-      if (item && item.classList.contains(ACTIVE_CLASS)) {
-        item.classList.remove(ACTIVE_CLASS);
-      }
-      let data = item.getAttribute('data-content');
-      if (data === selected) {
-        item.classList.add(ACTIVE_CLASS);
-      }
-    });
-  }
+    function updateActiveContent(selected) {
+      CONTENT.forEach((item) => {
+        if (item && item.classList.contains(ACTIVE_CLASS)) {
+          item.classList.remove(ACTIVE_CLASS);
+        }
+        let data = item.getAttribute('data-content');
+        if (data === selected) {
+          item.classList.add(ACTIVE_CLASS);
+        }
+      });
+    }
   </script>
   <style>
     .icon { border-bottom: 0px !important;}
@@ -46,7 +47,7 @@
               {% csrf_token %}
               <div class="field has-addons">
                 <div class="control is-expanded has-icons-left">
-                  <input id="query" name="query" class="input" type="text" placeholder="nickname">
+                  <input id="query_full" name="query_full" class="input" type="text" placeholder="nickname">
                   <span class="icon is-small is-left">
                     <i class="fas fa-magnifying-glass"></i>
                   </span>

core/templates/widgets/drilldown.html

@@ -0,0 +1,19 @@
+{% extends 'wm/widget.html' %}
+
+
+{% block widget_options %}
+  gs-w="5" gs-h="15"
+{% endblock %}
+
+{% block heading %}
+  Drilldown
+{% endblock %}
+
+{% block panel_content %}
+  {% include 'window-content/drilldown.html' %}
+{% endblock %}
+
+
+{% block custom_script_end %}
+  initTabs("{{ unique }}");
+{% endblock %}

core/templates/widgets/table_results.html

@@ -0,0 +1,43 @@
+{% extends 'wm/widget.html' %}
+{% load static %}
+
+{% block widget_options %}
+  gs-w="10" gs-h="1" gs-y="10" gs-x="1"
+{% endblock %}
+
+{% block heading %}
+  Results
+{% endblock %}
+
+{% block close_button %}
+  <i
+    class="fa-solid fa-xmark has-text-grey-light float-right"
+    onclick='grid.removeWidget("drilldown-widget-{{ unique }}"); //grid.compact();'></i>
+{% endblock %}
+
+{% block panel_content %}
+  {% include 'partials/notify.html' %}
+  <script src="{% static 'js/column-shifter.js' %}"></script>
+  {% if cache is not None %}
+    <span class="icon has-tooltip-bottom" data-tooltip="Cached">
+      <i class="fa-solid fa-database"></i>
+    </span>
+  {% endif %}
+
+  fetched {{ table.data|length }} hits in {{ took }}ms
+
+  {% if exemption is not None %}
+    <span class="icon has-tooltip-bottom" data-tooltip="God mode">
+      <i class="fa-solid fa-book-bible"></i>
+    </span>
+  {% else %}
+    {% if redacted is not None %}
+      <span class="icon has-tooltip-bottom" data-tooltip="{{ redacted }} redacted">
+        <i class="fa-solid fa-mask"></i>
+      </span>
+    {% endif %}
+  {% endif %}
+
+  {% include 'ui/drilldown/table_results_partial.html' %}
+  {% include 'ui/drilldown/sentiment_partial.html' %}
+{% endblock %}

core/templates/window-content/context.html

@@ -0,0 +1,122 @@
+{% load index %}
+{% load static %}
+
+<script src="{% static 'modal.js' %}"></script>
+<script>
+  document.addEventListener("restore-modal-scroll", function(event) {
+    var modalContent = document.getElementsByClassName("modal-content")[0];
+    var maxScroll = modalContent.scrollHeight - modalContent.offsetHeight;
+    var scrollpos = localStorage.getItem('scrollpos_modal_content');
+    if (scrollpos == 'BOTTOM') {
+      modalContent.scrollTop = maxScroll;
+    } else if (scrollpos) {
+      modalContent.scrollTop = scrollpos;
+    };
+  });
+
+  document.addEventListener("htmx:beforeSwap", function(event) {
+    var modalContent = document.getElementsByClassName("modal-content")[0];
+    var scrollpos = modalContent.scrollTop;
+    if(modalContent.scrollTop === (modalContent.scrollHeight - modalContent.offsetHeight)) {
+      localStorage.setItem('scrollpos_modal_content', 'BOTTOM');
+    } else {
+      localStorage.setItem('scrollpos_modal_content', scrollpos);
+    }
+
+  });
+</script>
+<style>
+  #tab-content-{{ unique }} div {
+    display: none;
+  }
+
+  #tab-content-{{ unique }} div.is-active {
+    display: block;
+  }
+</style>
+
+<div id="modal" class="modal is-active is-clipped">
+  <div class="modal-background"></div>
+  <div class="modal-content">
+    <div class="box">
+      {% include 'partials/notify.html' %}
+      <div class="tabs is-toggle is-fullwidth is-info" id="tabs-{{ unique }}">
+        <ul>
+          <li class="is-active" data-tab="1">
+            <a>
+              <span class="icon is-small"><i class="fa-solid fa-message-arrow-down"></i></span>
+              <span>Scrollback</span>
+            </a>
+          </li>
+          <li data-tab="2">
+            <a>
+              <span class="icon is-small"><i class="fa-solid fa-messages"></i></span>
+              <span>Context</span>
+            </a>
+          </li>
+          <li data-tab="3">
+            <a>
+              <span class="icon is-small"><i class="fa-solid fa-message"></i></span>
+              <span>Message</span>
+            </a>
+          </li>
+          <li data-tab="4">
+            <a>
+              <span class="icon is-small"><i class="fa-solid fa-asterisk"></i></span>
+              <span>Info</span>
+            </a>
+          </li>
+        </ul>
+      </div>
+      <div id="tab-content-{{ unique }}">
+        <div class="is-active" data-content="1">
+          <h4 class="subtitle is-4">Scrollback of {{ channel }} on {{ net }}{{ num }}</h4>
+          {% include 'modals/context_table.html' %}
+          {% if user.is_superuser and src == 'irc' %}
+            <form method="PUT">
+              <article class="field has-addons">
+                <article class="control is-expanded has-icons-left">
+                  <input id="context-input" name="msg" class="input" type="text" placeholder="Type your message here">
+                  <span class="icon is-small is-left">
+                    <i class="fas fa-magnifying-glass"></i>
+                  </span>
+                </article>
+                <article class="control">
+                  <article class="field">
+                    <button
+                      id="search"
+                      class="button is-info is-fullwidth"
+                      hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
+                      hx-put="{% url 'threshold_irc_msg' net num %}"
+                      hx-vals='{"channel": "{{ channel }}", "nick": "{{ nick }}"}'
+                      hx-trigger="click"
+                      hx-target="#context-input"
+                      hx-swap="outerHTML">
+                      Send
+                    </button>
+                  </article>
+                </article>
+              </article>
+            </form>
+          {% endif %}
+
+        </div>
+        <div data-content="2">
+          <h4 class="subtitle is-4">Scrollback of {{ channel }} on {{ net }}{{ num }} around {{ ts }}</h4>
+          Context
+        </div>
+        <div data-content="3">
+          <h4 class="subtitle is-4">Message details</h4>
+          Message deetails
+        </div>
+        <div data-content="4">
+          <h4 class="subtitle is-4">Information about {{ channel }} on {{ net }}{{ num }}</h4>
+          info
+        </div>
+      </div>
+    </div>
+    <script>initTabs("{{ unique }}");</script>
+    <button class="modal-close is-large" aria-label="close"></button>
+  </div>
+</div>
+

core/templates/window-content/context_table.html

@@ -0,0 +1,177 @@
+<article class="table-container" id="modal-context-table">
+  <table class="table is-fullwidth">
+    <thead>
+      <th></th>
+      <th></th>
+      <th></th>
+    </thead>
+    <tbody>
+      {% for item in object_list %}
+        {% if item.type == 'control' %}
+          <tr>
+            <td></td>
+            <td>
+              <span class="icon has-text-grey" data-tooltip="Hidden">
+                <i class="fa-solid fa-file-slash"></i>
+              </span>
+            </td>
+            <td>
+              <p class="has-text-grey">Hidden {{ item.hidden }} similar result{% if item.hidden > 1%}s{% endif %}</p>
+            </td>
+          </tr>
+        {% else %}
+          <tr>
+            <td>{{ item.time }}</td>
+            <td>
+              {% if item.type != 'znc' and item.type != 'self' and query is not True %}
+                <article class="nowrap-parent">
+                  <article class="nowrap-child">
+                    {% if item.type == 'msg' %}
+                      <span class="icon" data-tooltip="Message">
+                        <i class="fa-solid fa-message"></i>
+                      </span>
+                    {% elif item.type == 'join' %}
+                      <span class="icon" data-tooltip="Join">
+                        <i class="fa-solid fa-person-to-portal"></i>
+                      </span>
+                    {% elif item.type == 'part' %}
+                      <span class="icon" data-tooltip="Part">
+                        <i class="fa-solid fa-person-from-portal"></i>
+                      </span>
+                    {% elif item.type == 'quit' %}
+                      <span class="icon" data-tooltip="Quit">
+                        <i class="fa-solid fa-circle-xmark"></i>
+                      </span>
+                    {% elif item.type == 'kick' %}
+                      <span class="icon" data-tooltip="Kick">
+                        <i class="fa-solid fa-user-slash"></i>
+                      </span>
+                    {% elif item.type == 'nick' %}
+                      <span class="icon" data-tooltip="Nick">
+                        <i class="fa-solid fa-signature"></i>
+                      </span>
+                    {% elif item.type == 'mode' %}
+                      <span class="icon" data-tooltip="Mode">
+                        <i class="fa-solid fa-gear"></i>
+                      </span>
+                    {% elif item.type == 'action' %}
+                      <span class="icon" data-tooltip="Action">
+                        <i class="fa-solid fa-exclamation"></i>
+                      </span>
+                    {% elif item.type == 'notice' %}
+                      <span class="icon" data-tooltip="Notice">
+                        <i class="fa-solid fa-message-code"></i>
+                      </span>
+                    {% elif item.type == 'conn' %}
+                      <span class="icon" data-tooltip="Connection">
+                        <i class="fa-solid fa-cloud-exclamation"></i>
+                      </span>
+                    {% elif item.type == 'znc' %}
+                      <span class="icon" data-tooltip="ZNC">
+                        <i class="fa-brands fa-unity"></i>
+                      </span>
+                    {% elif item.type == 'query' %}
+                      <span class="icon" data-tooltip="Query">
+                        <i class="fa-solid fa-message"></i>
+                      </span>
+                    {% elif item.type == 'highlight' %}
+                      <span class="icon" data-tooltip="Highlight">
+                        <i class="fa-solid fa-exclamation"></i>
+                      </span>
+                    {% elif item.type == 'who' %}
+                      <span class="icon" data-tooltip="Who">
+                        <i class="fa-solid fa-passport"></i>
+                      </span>
+                    {% elif item.type == 'topic' %}
+                      <span class="icon" data-tooltip="Topic">
+                        <i class="fa-solid fa-sign"></i>
+                      </span>
+                    {% else %}
+                      {{ item.type }}
+                    {% endif %}
+                    {% if item.online is True %}
+                      <span class="icon has-text-success has-tooltip-success" data-tooltip="Online">
+                        <i class="fa-solid fa-circle"></i>
+                      </span>
+                    {% elif item.online is False %}
+                      <span class="icon has-text-danger has-tooltip-danger" data-tooltip="Offline">
+                        <i class="fa-solid fa-circle"></i>
+                      </span>
+                    {% else %}
+                      <span class="icon has-text-warning has-tooltip-warning" data-tooltip="Unknown">
+                        <i class="fa-solid fa-circle"></i>
+                      </span>
+                    {% endif %}
+                    {% if item.src == 'irc' %}
+                      <a
+                        hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
+                        hx-post="{% url 'modal_drilldown' %}"
+                        hx-vals='{"net": "{{ item.net|escapejs }}", "nick": "{{ item.nick|escapejs }}", "channel": "{{ item.channel|escapejs }}"}'
+                        hx-target="#modals-here"
+                        hx-trigger="click"
+                        class="has-text-black">
+                        <span class="icon" data-tooltip="Open drilldown modal">
+                          <i class="fa-solid fa-album"></i>
+                        </span>
+                      </a>
+                    {% endif %}
+                  </article>
+                  <a class="nowrap-child has-text-link is-underlined" onclick="populateSearch('nick', '{{ item.nick|escapejs }}')">
+                    {{ item.nick }}
+                  </a>
+                  {% if item.num_chans != '—' %}
+                    <article class="nowrap-child">
+                      <span class="tag">
+                        {{ item.num_chans }}
+                      </span>
+                    </article>
+                  {% endif %}
+                </article>
+              {% endif %}
+              {% if item.type == 'self' %}
+                <span class="icon has-text-primary" data-tooltip="You">
+                  <i class="fa-solid fa-message-check"></i>
+                </span>
+              {% elif item.type == 'znc' %}
+                <span class="icon has-text-info" data-tooltip="ZNC">
+                  <i class="fa-brands fa-unity"></i>
+                </span>
+              {% elif query %}
+                <span class="icon has-text-info" data-tooltip="Auth">
+                  <i class="fa-solid fa-passport"></i>
+                </span>
+              {% endif %}
+            </td>
+            <td class="wrap">{{ item.msg }}</td>
+          </tr>
+        {% endif %}
+      {% endfor %}
+    </tbody>
+  </table>
+  {% if object_list %}
+    <div
+      class="modal-refresh"
+      style="display: none;"
+      hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
+      hx-post="{% url 'modal_context_table' %}"
+      hx-vals='{"net": "{{ net }}",
+      "num": "{{ num }}",
+      "src": "{{ src }}",
+      "channel": "{{ channel }}",
+      "time": "{{ time }}",
+      "date": "{{ date }}",
+      "index": "{{ index }}",
+      "type": "{{ type }}",
+      "mtype": "{{ mtype }}",
+      "nick": "{{ nick }}",
+      "dedup": "{{ params.dedup }}"}'
+      hx-target="#modal-context-table"
+      hx-trigger="every 5s">
+    </div>
+  {% endif %}
+</article>
+
+<script>
+  var modal_event = new Event('restore-modal-scroll');
+  document.dispatchEvent(modal_event);
+</script>

core/templates/window-content/drilldown.html

@@ -0,0 +1,107 @@
+{% load index %}
+{% load static %}
+<style>
+  #tab-content-{{ unique }} div {
+    display: none;
+  }
+
+  #tab-content-{{ unique }} div.is-active {
+    display: block;
+  }
+</style>
+<div class="tabs is-toggle is-fullwidth is-info" id="tabs-{{ unique }}">
+  <ul>
+    <li class="is-active" data-tab="1">
+      <a>
+        <span class="icon is-small"><i class="fa-solid fa-user"></i></span>
+        <span>Channels</span>
+      </a>
+    </li>
+    <li data-tab="2">
+      <a>
+        <span class="icon is-small"><i class="fa-solid fa-hashtag"></i></span>
+        <span>Users</span>
+      </a>
+    </li>
+    <li data-tab="3">
+      <a>
+        <span class="icon is-small"><i class="fa-solid fa-people"></i></span>
+        <span>Intersection</span>
+      </a>
+    </li>
+    <li data-tab="4">
+      <a>
+        <span class="icon is-small"><i class="fa-solid fa-hashtag"></i></span>
+        <span>Intersection</span>
+      </a>
+    </li>
+  </ul>
+</div>
+<div id="tab-content-{{ unique }}">
+  <div class="is-active" data-content="1">
+    <h4 class="subtitle is-4">Channels for {{ nick }} on {{ net }}</h4>
+    {% for channel in chans %}
+      <a class="panel-block is-active">
+        <span class="panel-icon">
+          <i class="fa-solid fa-hashtag" aria-hidden="true"></i>
+        </span>
+        {{ channel }}
+        {% if channel in num_users %}
+          <span class="tag">
+            {{ num_users|index:channel }}
+          </span>
+        {% endif %}
+      </a>
+    {% endfor %}
+  </div>
+  <div data-content="2">
+    <h4 class="subtitle is-4">Users on {{ channel }} for {{ net }}</h4>
+    {% for user in users %}
+      <a class="panel-block is-active">
+        <span class="panel-icon">
+          <i class="fa-solid fa-user" aria-hidden="true"></i>
+        </span>
+        {{ user }}
+        {% if user in num_chans %}
+          <span class="tag">
+            {{ num_chans|index:user }}
+          </span>
+        {% endif %}
+      </a>
+    {% endfor %}
+  </div>
+  <div data-content="3">
+    <h4 class="subtitle is-4">Users sharing channels with {{ nick }} on {{ net }}</h4>
+    {% for user in inter_users %}
+      <a class="panel-block is-active">
+        <span class="panel-icon">
+          <i class="fa-solid fa-user" aria-hidden="true"></i>
+        </span>
+        {{ user }}
+        {% if user in num_chans %}
+          <span class="tag">
+            {{ num_chans|index:user }}
+          </span>
+        {% endif %}
+      </a>
+    {% endfor %}
+  </div>
+  <div data-content="4">
+    <h4 class="subtitle is-4">Channels sharing users with {{ channel }} on {{ net }}</h4>
+    {% for channel in inter_chans %}
+      <a class="panel-block is-active">
+        <span class="panel-icon">
+          <i class="fa-solid fa-hashtag" aria-hidden="true"></i>
+        </span>
+        {{ channel }}
+        {% if channel in num_users %}
+          <span class="tag">
+            {{ num_users|index:channel }}
+          </span>
+        {% endif %}
+      </a>
+    {% endfor %}
+  </div>
+  <script>initTabs("{{ unique }}");</script>
+</div>
+

core/templates/windows/drilldown.html

@@ -0,0 +1,9 @@
+{% extends 'wm/magnet.html' %}
+
+{% block heading %}
+  Drilldown
+{% endblock %}
+
+{% block panel_content %}
+  {% include 'window-content/drilldown.html' %}
+{% endblock %}

core/templates/wm/magnet.html

@@ -0,0 +1,8 @@
+<magnet-block attract-distance="10" align-to="outer|center" class="floating-window">
+  {% extends 'wm/panel.html' %}
+  {% block heading %}
+  {% endblock %}
+
+  {% block panel_content %}
+  {% endblock %}
+</magnet-block>

core/templates/wm/modal.html

@@ -0,0 +1,19 @@
+{% load static %}
+
+<script src="{% static 'modal.js' %}"></script>
+{% block scripts %}
+{% endblock %}
+
+{% block styles %}
+{% endblock %}
+
+<div id="modal" class="modal is-active is-clipped">
+  <div class="modal-background"></div>
+  <div class="modal-content">
+    <div class="box">
+      {% block modal_content %}
+      {% endblock %}
+      <button class="modal-close is-large" aria-label="close"></button>
+    </div>
+  </div>
+</div>

core/templates/wm/panel.html

@@ -0,0 +1,19 @@
+
+<nav class="panel">
+  <p class="panel-heading" style="padding: .2em; line-height: .5em;">
+    <i class="fa-solid fa-arrows-up-down-left-right has-text-grey-light"></i>
+    {% block close_button %}
+      <i
+        class="fa-solid fa-xmark has-text-grey-light float-right"
+        data-script="on click remove the closest <nav/>"></i>
+    {% endblock %}
+    {% block heading %}
+    {% endblock %}
+  </p>
+  <article class="panel-block is-active">
+    <div class="control">
+      {% block panel_content %}
+      {% endblock %}
+    </div>
+  </article>
+</nav>

core/templates/wm/widget.html

@@ -0,0 +1,37 @@
+<div id="drilldown-widget">
+  <div id="drilldown-widget-{{ unique }}" class="grid-stack-item" {% block widget_options %}{% endblock %}>
+    <div class="grid-stack-item-content">
+
+      <nav class="panel">
+        <p class="panel-heading" style="padding: .2em; line-height: .5em;">
+          <i class="fa-solid fa-arrows-up-down-left-right has-text-grey-light"></i>
+          {% block close_button %}
+            <i
+              class="fa-solid fa-xmark has-text-grey-light float-right"
+              onclick='grid.removeWidget("drilldown-widget-{{ unique }}");'></i>
+          {% endblock %}
+          <i
+            class="fa-solid fa-arrows-minimize has-text-grey-light float-right"
+            onclick='grid.compact();'></i>
+          {% block heading %}
+          {% endblock %}
+        </p>
+        <article class="panel-block is-active">
+          <div class="control">
+            {% block panel_content %}
+            {% endblock %}
+          </div>
+        </article>
+      </nav>
+    </div>
+  </div>
+</div>
+
+<script>
+  {% block custom_script %}
+  {% endblock %}
+  var widget_event = new Event('load-widget');
+  document.dispatchEvent(widget_event);
+</script>
+{% block custom_end %}
+{% endblock %}

core/templatetags/urlsafe.py

@@ -0,0 +1,10 @@
+import urllib.parse
+
+from django import template
+
+register = template.Library()
+
+
+@register.filter
+def urlsafe(h):
+    return urllib.parse.quote(h, safe="")

core/util/logs.py

@@ -0,0 +1,69 @@
+# Other library imports
+import logging
+
+log = logging.getLogger("util")
+
+debug = True
+
+# Color definitions
+BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = range(8)
+COLORS = {
+    "WARNING": YELLOW,
+    "INFO": WHITE,
+    "DEBUG": BLUE,
+    "CRITICAL": YELLOW,
+    "ERROR": RED,
+}
+RESET_SEQ = "\033[0m"
+COLOR_SEQ = "\033[1;%dm"
+BOLD_SEQ = "\033[1m"
+
+
+def formatter_message(message, use_color=True):
+    if use_color:
+        message = message.replace("$RESET", RESET_SEQ).replace("$BOLD", BOLD_SEQ)
+    else:
+        message = message.replace("$RESET", "").replace("$BOLD", "")
+    return message
+
+
+class ColoredFormatter(logging.Formatter):
+    def __init__(self, msg, use_color=True):
+        logging.Formatter.__init__(self, msg)
+        self.use_color = use_color
+
+    def format(self, record):
+        levelname = record.levelname
+        if self.use_color and levelname in COLORS:
+            levelname_color = (
+                COLOR_SEQ % (30 + COLORS[levelname]) + levelname + RESET_SEQ
+            )
+            record.levelname = levelname_color
+        return logging.Formatter.format(self, record)
+
+
+def get_logger(name):
+
+    # Define the logging format
+    FORMAT = "%(asctime)s %(levelname)18s $BOLD%(name)13s$RESET  - %(message)s"
+    COLOR_FORMAT = formatter_message(FORMAT, True)
+    color_formatter = ColoredFormatter(COLOR_FORMAT)
+    # formatter = logging.Formatter(
+
+    # Why is this so complicated?
+    ch = logging.StreamHandler()
+    ch.setLevel(logging.INFO)
+    # ch.setFormatter(formatter)
+    ch.setFormatter(color_formatter)
+
+    # Define the logger on the base class
+    log = logging.getLogger(name)
+    log.setLevel(logging.INFO)
+    if debug:
+        log.setLevel(logging.DEBUG)
+        ch.setLevel(logging.DEBUG)
+
+    # Add the handler and stop it being silly and printing everything twice
+    log.addHandler(ch)
+    log.propagate = False
+    return log

core/views/helpers.py

@@ -0,0 +1,324 @@
+# import re
+# from base64 import b64encode
+# from random import randint
+
+# from cryptography.hazmat.primitives.ciphers import Cipher, algorithms
+# from cryptography.hazmat.primitives.ciphers.modes import ECB
+# from django.conf import settings
+# from siphashc import siphash
+# from sortedcontainers import SortedSet
+
+# from core import r
+from django.conf import settings
+
+
+class SearchDenied:
+    def __init__(self, key, value):
+        self.key = key
+        self.value = value
+
+
+class LookupDenied:
+    def __init__(self, key, value):
+        self.key = key
+        self.value = value
+
+
+def remove_defaults(query_params):
+    for field, value in list(query_params.items()):
+        if field in settings.DRILLDOWN_DEFAULT_PARAMS:
+            if value == settings.DRILLDOWN_DEFAULT_PARAMS[field]:
+                del query_params[field]
+
+
+def add_defaults(query_params):
+    for field, value in settings.DRILLDOWN_DEFAULT_PARAMS.items():
+        if field not in query_params:
+            query_params[field] = value
+
+
+def dedup_list(data, check_keys):
+    """
+    Remove duplicate dictionaries from list.
+    """
+    seen = set()
+    out = []
+
+    dup_count = 0
+    for x in data:
+        dedupeKey = tuple(x[k] for k in check_keys if k in x)
+        if dedupeKey in seen:
+            dup_count += 1
+            continue
+        if dup_count > 0:
+            out.append({"type": "control", "hidden": dup_count})
+            dup_count = 0
+        out.append(x)
+        seen.add(dedupeKey)
+    if dup_count > 0:
+        out.append({"type": "control", "hidden": dup_count})
+    return out
+
+
+# from random import randint
+# from timeit import timeit
+# entries = 10000
+# a = [
+#     {'ts': "sss", 'msg': randint(1, 2), str(randint(1, 2)): \
+#       randint(1, 2)} for x in range(entries)
+#     ]
+# kk = ["msg", "nick"]
+# call = lambda: dedup_list(a, kk)
+# #print(timeit(call, number=10))
+# print(dedup_list(a, kk))
+
+# # sh-5.1$ python helpers.py
+# # 1.0805372429895215
+
+
+# def base36encode(number, alphabet="0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"):
+#     """Converts an integer to a base36 string."""
+#     if not isinstance(number, (int)):
+#         raise TypeError("number must be an integer")
+
+#     base36 = ""
+#     sign = ""
+
+#     if number < 0:
+#         sign = "-"
+#         number = -number
+
+#     if 0 <= number < len(alphabet):
+#         return sign + alphabet[number]
+
+#     while number != 0:
+#         number, i = divmod(number, len(alphabet))
+#         base36 = alphabet[i] + base36
+
+#     return sign + base36
+
+
+# def base36decode(number):
+#     return int(number, 36)
+
+
+# def randomise_list(user, data):
+#     """
+#     Randomise data in a list of dictionaries.
+#     """
+#     if user.has_perm("core.bypass_randomisation"):
+#         return
+#     if isinstance(data, list):
+#         for index, item in enumerate(data):
+#             for key, value in item.items():
+#                 if key in settings.RANDOMISE_FIELDS:
+#                     if isinstance(value, int):
+#                         min_val = value - (value * settings.RANDOMISE_RATIO)
+#                         max_val = value + (value * settings.RANDOMISE_RATIO)
+#                         new_val = randint(int(min_val), int(max_val))
+#                         data[index][key] = new_val
+#     elif isinstance(data, dict):
+#         for key, value in data.items():
+#             # if key in settings.RANDOMISE_FIELDS:
+#             if isinstance(value, int):
+#                 min_val = value - (value * settings.RANDOMISE_RATIO)
+#                 max_val = value + (value * settings.RANDOMISE_RATIO)
+#                 new_val = randint(int(min_val), int(max_val))
+#                 data[key] = new_val
+
+
+# def obfuscate_list(user, data):
+#     """
+#     Obfuscate data in a list of dictionaries.
+#     """
+#     if user.has_perm("core.bypass_obfuscation"):
+#         return
+#     for index, item in enumerate(data):
+#         for key, value in item.items():
+#             # Obfuscate a ratio of the field
+#             if key in settings.OBFUSCATE_FIELDS:
+#                 length = len(value) - 1
+#                 split = int(length * settings.OBFUSCATE_KEEP_RATIO)
+#                 first_part = value[:split]
+#                 second_part = value[split:]
+#                 second_len = len(second_part)
+#                 second_part = "*" * second_len
+#                 data[index][key] = first_part + second_part
+#             # Obfuscate value based on fields
+#             # Example: 2022-02-02 -> 2022-02-**
+#             # 14:11:12 -> 14:11:**
+#             elif key in settings.OBFUSCATE_FIELDS_SEP:
+#                 if "-" in value:
+#                     sep = "-"
+#                     value_spl = value.split("-")
+#                     hide_num = settings.OBFUSCATE_DASH_NUM
+#                 elif ":" in value:
+#                     sep = ":"
+#                     value_spl = value.split(":")
+#                     hide_num = settings.OBFUSCATE_COLON_NUM
+
+#                 first_part = value_spl[:hide_num]
+#                 second_part = value_spl[hide_num:]
+#                 for index_x, x in enumerate(second_part):
+#                     x_len = len(x)
+#                     second_part[index_x] = "*" * x_len
+#                 result = sep.join([*first_part, *second_part])
+#                 data[index][key] = result
+#     for key in settings.COMBINE_FIELDS:
+#         for index, item in enumerate(data):
+#             if key in item:
+#                 k1, k2 = settings.COMBINE_FIELDS[key]
+#                 if k1 in item and k2 in item:
+#                     data[index][key] = item[k1] + item[k2]
+
+
+# def hash_list(user, data, hash_keys=False):
+#     """
+#     Hash a list of dicts or a list with SipHash42.
+#     """
+#     if user.has_perm("core.bypass_hashing"):
+#         return
+#     cache = "cache.hash"
+#     hash_table = {}
+#     if isinstance(data, dict):
+#         data_copy = [{x: data[x]} for x in data]
+#     else:
+#         data_copy = type(data)((data))
+#     for index, item in enumerate(data_copy):
+#         if "src" in item:
+#             if item["src"] in settings.SAFE_SOURCES:
+#                 continue
+#         if isinstance(item, dict):
+#             for key, value in list(item.items()):
+#                 if (
+#                     key not in settings.WHITELIST_FIELDS
+#                     and key not in settings.NO_OBFUSCATE_PARAMS
+#                 ):
+#                     if isinstance(value, int):
+#                         value = str(value)
+#                     if isinstance(value, bool):
+#                         continue
+#                     if value is None:
+#                         continue
+#                     if hash_keys:
+#                         hashed = siphash(settings.HASHING_KEY, key)
+#                     else:
+#                         hashed = siphash(settings.HASHING_KEY, value)
+#                     encoded = base36encode(hashed)
+#                     if encoded not in hash_table:
+#                         if hash_keys:
+#                             hash_table[encoded] = key
+#                         else:
+#                             hash_table[encoded] = value
+#                     if hash_keys:
+#                         # Rename the dict key
+#                         data[encoded] = data.pop(key)
+#                     else:
+#                         data[index][key] = encoded
+#         elif isinstance(item, str):
+#             hashed = siphash(settings.HASHING_KEY, item)
+#             encoded = base36encode(hashed)
+#             if encoded not in hash_table:
+#                 hash_table[encoded] = item
+#             data[index] = encoded
+#     if hash_table:
+#         r.hmset(cache, hash_table)
+
+
+# def hash_lookup(user, data_dict, supplementary_data=None):
+#     cache = "cache.hash"
+#     hash_list = SortedSet()
+#     denied = []
+#     for key, value in list(data_dict.items()):
+#         if "source" in data_dict:
+#             if data_dict["source"] in settings.SAFE_SOURCES:
+#                 continue
+#         if "src" in data_dict:
+#             if data_dict["src"] in settings.SAFE_SOURCES:
+#                 continue
+#         if supplementary_data:
+#             if "source" in supplementary_data:
+#                 if supplementary_data["source"] in settings.SAFE_SOURCES:
+#                     continue
+#         if key in settings.SEARCH_FIELDS_DENY:
+#             if not user.has_perm("core.bypass_hashing"):
+#                 data_dict[key] = SearchDenied(key=key, value=data_dict[key])
+#                 denied.append(data_dict[key])
+#         if (
+#             key not in settings.WHITELIST_FIELDS
+#             and key not in settings.NO_OBFUSCATE_PARAMS
+#         ):
+#             if not value:
+#                 continue
+#             # hashes = re.findall("\|([^\|]*)\|", value)  # noqa
+#             if isinstance(value, str):
+#                 hashes = re.findall("[A-Z0-9]{12,13}", value)
+#             elif isinstance(value, dict):
+#                 hashes = []
+#                 for key, value in value.items():
+#                     if not value:
+#                         continue
+#                     hashes_iter = re.findall("[A-Z0-9]{12,13}", value)
+#                     for h in hashes_iter:
+#                         hashes.append(h)
+#             if not hashes:
+#                 # Otherwise the user could inject plaintext search queries
+#                 if not user.has_perm("core.bypass_hashing"):
+#                     data_dict[key] = SearchDenied(key=key, value=data_dict[key])
+#                     denied.append(data_dict[key])
+#                     continue
+#             else:
+#                 # There are hashes here but there shouldn't be!
+#                 if key in settings.TAG_SEARCH_DENY:
+#                     data_dict[key] = LookupDenied(key=key, value=data_dict[key])
+#                     denied.append(data_dict[key])
+#                     continue
+
+#             for hash in hashes:
+#                 hash_list.add(hash)
+
+#     if hash_list:
+#         values = r.hmget(cache, *hash_list)
+#         if not values:
+#             return
+#         for index, val in enumerate(values):
+#             if val is None:
+#                 values[index] = b"ERR"
+#         values = [x.decode() for x in values]
+#         total = dict(zip(hash_list, values))
+#         for key in data_dict.keys():
+#             for hash in total:
+#                 if data_dict[key]:
+#                     if isinstance(data_dict[key], str):
+#                         if hash in data_dict[key]:
+#                             data_dict[key] = data_dict[key].replace(
+#                                 f"{hash}", total[hash]
+#                             )
+#                     elif isinstance(data_dict[key], dict):
+#                         for k2, v2 in data_dict[key].items():
+#                             if hash in v2:
+#                                 data_dict[key][k2] = v2.repl
+# ace(f"{hash}", total[hash])
+#     return denied
+
+
+# def encrypt_list(user, data, secret):
+#     if user.has_perm("core.bypass_encryption"):
+#         return
+#     cipher = Cipher(algorithms.AES(secret), ECB())
+#     for index, item in enumerate(data):
+#         for key, value in item.items():
+#             if key not in settings.WHITELIST_FIELDS:
+#                 encryptor = cipher.encryptor()
+#                 if isinstance(value, int):
+#                     value = str(value)
+#                 if isinstance(value, bool):
+#                     continue
+#                 if value is None:
+#                     continue
+#                 decoded = value.encode("utf8", "replace")
+#                 length = 16 - (len(decoded) % 16)
+#                 decoded += bytes([length]) * length
+#                 ct = encryptor.update(decoded) + encryptor.finalize()
+#                 final_str = b64encode(ct)
+#                 data[index][key] = final_str.decode("utf-8", "replace")

core/views/manage/threshold/irc.py

@@ -71,7 +71,7 @@ class ThresholdIRCNetworkInfoEdit(SuperUserRequiredMixin, APIView):
         Return the form to edit a network.
         """
         network = threshold.get_irc_network(net)
-        editable = ["auth", "host", "last", "port", "security"]
+        editable = ["auth", "host", "last", "port", "security", "chanlimit"]
         context = {
             "net": net,
             "network": {k: v for k, v in network.items() if k in editable},
@@ -104,7 +104,11 @@ class ThresholdIRCNetworkRelays(SuperUserRequiredMixin, View):
 
     def get(self, request, net):
         relays = threshold.get_irc_relays(net)
-        context = {"relays": relays["relays"]}
+        sinst = threshold.get_irc_sinst(net)
+        context = {"net": net, "relays": relays["relays"]}
+        if sinst:
+            if sinst["success"]:
+                context["sinst"] = sinst["sinst"]
         return render(request, self.template_name, context)
 
 
@@ -124,12 +128,74 @@ class ThresholdIRCNetworkRelayDel(SuperUserRequiredMixin, APIView):
             message = deleted["reason"]
             message_class = "danger"
         relays = threshold.get_irc_relays(net)
+        sinst = threshold.get_irc_sinst(net)
         context = {
             "net": net,
             "message": message,
             "class": message_class,
             "relays": relays["relays"],
         }
+        if sinst:
+            if sinst["success"]:
+                context["sinst"] = sinst["sinst"]
+        return render(request, self.template_name, context)
+
+
+class ThresholdIRCNetworkRelayProvision(SuperUserRequiredMixin, APIView):
+    template_name = "manage/threshold/irc/network/relays.html"
+
+    def post(self, request, net, num):
+        """
+        Provision a relay
+        """
+        provisioned = threshold.irc_provision_relay(net, num)
+        if provisioned["success"]:
+
+            message = f"Provisioned relay {num}"
+            message_class = "success"
+        else:
+            message = provisioned["reason"]
+            message_class = "danger"
+        relays = threshold.get_irc_relays(net)
+        sinst = threshold.get_irc_sinst(net)
+        context = {
+            "net": net,
+            "message": message,
+            "class": message_class,
+            "relays": relays["relays"],
+        }
+        if sinst:
+            if sinst["success"]:
+                context["sinst"] = sinst["sinst"]
+        return render(request, self.template_name, context)
+
+
+class ThresholdIRCNetworkRelayAuth(SuperUserRequiredMixin, APIView):
+    template_name = "manage/threshold/irc/network/relays.html"
+
+    def post(self, request, net, num):
+        """
+        Provision a relay
+        """
+        provisioned = threshold.irc_enable_auth(net, num)
+        if provisioned["success"]:
+
+            message = f"Enabled authentication on relay {num}"
+            message_class = "success"
+        else:
+            message = provisioned["reason"]
+            message_class = "danger"
+        relays = threshold.get_irc_relays(net)
+        sinst = threshold.get_irc_sinst(net)
+        context = {
+            "net": net,
+            "message": message,
+            "class": message_class,
+            "relays": relays["relays"],
+        }
+        if sinst:
+            if sinst["success"]:
+                context["sinst"] = sinst["sinst"]
         return render(request, self.template_name, context)
 
 
@@ -151,11 +217,71 @@ class ThresholdIRCNetworkRelayStatus(SuperUserRequiredMixin, APIView):
                 message = "Error updating status"
             message_class = "danger"
         relays = threshold.get_irc_relays(net)
+        sinst = threshold.get_irc_sinst(net)
         context = {
             "relays": relays["relays"],
             "message": message,
             "class": message_class,
         }
+        if sinst:
+            if sinst["success"]:
+                context["sinst"] = sinst["sinst"]
+        return render(request, self.template_name, context)
+
+
+class ThresholdIRCNetworkChannelsAPI(SuperUserRequiredMixin, APIView):
+    template_name = "manage/threshold/irc/network/channels.html"
+    parser_classes = [FormParser]
+
+    def delete(self, request, net):
+        """
+        Part a channel.
+        :param net: network name
+        :param channel: channel name
+        """
+        channel = request.data["channel"]
+        parted = threshold.part_channel(net, channel)
+        if parted["success"]:
+            message = f"Requested part on relays: {', '.join(parted['relays'])}"
+            message_class = "success"
+        else:
+            message = parted["reason"]
+            message_class = "danger"
+
+        channels = threshold.get_irc_channels(net)
+        context = {
+            "net": net,
+            "channels": channels["channels"],
+            "message": message,
+            "class": message_class,
+        }
+        return render(request, self.template_name, context)
+
+    def put(self, request, net):
+        """
+        Join a channel.
+        :param net: network name
+        """
+        if "channel" not in request.data:
+            message = "No channel specified"
+            message_class = "danger"
+        else:
+            channel = request.data["channel"]
+            joined = threshold.join_channel(net, channel)
+            if joined["success"]:
+                message = f"Requested join on relay: {joined['relays']}"
+                message_class = "success"
+            else:
+                message = joined["reason"]
+                message_class = "danger"
+
+        channels = threshold.get_irc_channels(net)
+        context = {
+            "net": net,
+            "channels": channels["channels"],
+            "message": message,
+            "class": message_class,
+        }
         return render(request, self.template_name, context)
 
 
@@ -286,6 +412,123 @@ class ThresholdIRCActionsAddNetwork(SuperUserRequiredMixin, APIView):
         return render(request, template_name, context)
 
 
+class ThresholdIRCActionsRegistration(SuperUserRequiredMixin, APIView):
+    template_name = "manage/threshold/irc/network/modals/registration.html"
+    parser_classes = [FormParser]
+
+    def get(self, request, net=None):
+        """
+        Get registration modal
+        """
+        message = None
+        message_class = None
+        if net == "None":
+            net = None
+        unreg = threshold.irc_get_unreg(net)
+        if not unreg:
+            message = "Could not get registration status."
+            message_class = "danger"
+        elif not unreg["success"]:
+            if "reason" in unreg:
+                message = unreg["reason"]
+                message_class = "danger"
+            else:
+                message = "Getting registration status failed."
+                message_class = "danger"
+        context = {
+            "net": net,
+            "message": message,
+            "class": message_class,
+        }
+        if "unreg" in unreg:
+            context["unreg"] = unreg["unreg"]
+        return render(request, self.template_name, context)
+
+    def put(self, request, net=None):
+        """
+        Confirm registration for networks.
+        """
+        message = None
+        message_class = None
+        if net == "None":
+            net = None
+        if request.resolver_match.url_name == "threshold_irc_actions_registration_net":
+            template_name = "manage/threshold/irc/network/actions.html"
+        else:
+            template_name = "manage/threshold/irc/overview/actions.html"
+        updated = threshold.irc_confirm_accounts(request.data)
+
+        message = "Registration confirmed successfully."
+        message_class = "success"
+
+        if not updated["success"]:
+            message = updated["reason"]
+            message_class = "danger"
+        unreg = threshold.irc_get_unreg(net)
+        if not unreg:
+            message = "Could not get registration status."
+            message_class = "danger"
+        elif not unreg["success"]:
+            if "reason" in unreg:
+                message = unreg["reason"]
+                message_class = "danger"
+            else:
+                message = "Getting registration status failed."
+                message_class = "danger"
+        context = {
+            "net": net,
+            "message": message,
+            "class": message_class,
+        }
+        if "unreg" in unreg:
+            context["unreg"] = unreg["unreg"]
+        return render(request, template_name, context)
+
+
+class ThresholdIRCActionsRegistrationAuth(SuperUserRequiredMixin, APIView):
+    template_name = "manage/threshold/irc/network/modals/registration.html"
+    parser_classes = [FormParser]
+
+    def post(self, request):
+        """
+        Confirm registration for networks.
+        """
+        updated = threshold.irc_check_auth(request.data)
+
+        message = "Re-checked authentication successfully."
+        message_class = "success"
+
+        if not updated["success"]:
+            message = updated["reason"]
+            message_class = "danger"
+        if "net" in request.data:
+            if request.data["net"] != "None":
+                net = request.data["net"]
+            else:
+                net = None
+        else:
+            net = None
+        unreg = threshold.irc_get_unreg(net)
+        if not unreg:
+            message = "Could not get registration status."
+            message_class = "danger"
+        elif not unreg["success"]:
+            if "reason" in unreg:
+                message = unreg["reason"]
+                message_class = "danger"
+            else:
+                message = "Getting registration status failed."
+                message_class = "danger"
+        context = {
+            "net": net,
+            "message": message,
+            "class": message_class,
+        }
+        if "unreg" in unreg:
+            context["unreg"] = unreg["unreg"]
+        return render(request, self.template_name, context)
+
+
 class ThresholdIRCAliasesEdit(SuperUserRequiredMixin, APIView):
     template_name = "manage/threshold/irc/overview/modals/edit-aliases.html"
     parser_classes = [FormParser]
@@ -383,7 +626,7 @@ class ThresholdIRCNetworkActionsAuto(SuperUserRequiredMixin, View):
         return render(request, self.template_name, context)
 
 
-class ThresholdIRCNetworkActionsList(SuperUserRequiredMixin, View):
+class ThresholdIRCNetworkList(SuperUserRequiredMixin, View):
     template_name = "manage/threshold/irc/network/actions.html"
 
     def post(self, request, net):
@@ -408,6 +651,28 @@ class ThresholdIRCNetworkActionsList(SuperUserRequiredMixin, View):
         }
         return render(request, self.template_name, context)
 
+    def get(self, request, net):
+        """
+        Get list results from network.
+        """
+        template_name = "manage/threshold/irc/network/stats.html"
+        listinfo = threshold.get_irc_list_info(net)
+        if not listinfo["success"]:
+            if "reason" in listinfo:
+                message = listinfo["reason"]
+            else:
+                message = "Could not get list info."
+            message = listinfo["reason"]
+            message_class = "danger"
+            context = {"message": message, "class": message_class}
+            return render(request, template_name, context)
+
+        context = {
+            "net": net,
+            "list": listinfo["listinfo"],
+        }
+        return render(request, template_name, context)
+
 
 class ThresholdIRCNetworkActionsRelay(SuperUserRequiredMixin, APIView):
     template_name = "manage/threshold/irc/network/actions.html"
@@ -442,3 +707,56 @@ class ThresholdIRCNetworkActionsRelay(SuperUserRequiredMixin, APIView):
             "class": message_class,
         }
         return render(request, self.template_name, context)
+
+
+class ThresholdIRCSendMessage(SuperUserRequiredMixin, APIView):
+    parser_classes = [FormParser]
+    template_name = "partials/context-input.html"
+
+    def put(self, request, net, num):
+        """
+        Send a message
+        """
+        if "msg" not in request.data:
+            message = "No message to send"
+            message_class = "danger"
+            return render(
+                request,
+                self.template_name,
+                {"message": message, "class": message_class},
+            )
+        if "channel" not in request.data:
+            message = "No channel"
+            message_class = "danger"
+            return render(
+                request,
+                self.template_name,
+                {"message": message, "class": message_class},
+            )
+        nick = None
+        if "nick" in request.data:
+            nick = request.data["nick"]
+        if nick:
+            messaged = threshold.send_irc_message(
+                net, num, request.data["channel"], request.data["msg"], nick=nick
+            )
+        else:
+            messaged = threshold.send_irc_message(
+                net, num, request.data["channel"], request.data["msg"]
+            )
+        if not messaged:
+            message = "Failed to send message"
+            message_class = "danger"
+        elif messaged["success"]:
+            message = "Send message"
+            message_class = "success"
+        else:
+            message = messaged["reason"]
+            message_class = "danger"
+
+        context = {
+            "net": net,
+            "message": message,
+            "class": message_class,
+        }
+        return render(request, self.template_name, context)

core/views/ui/drilldown.py

@@ -1,47 +1,28 @@
-import json
 import urllib
+import uuid
 
+import ujson
 from django.conf import settings
 from django.http import HttpResponse, JsonResponse
 from django.shortcuts import render
 from django.urls import reverse
 from django.views import View
-from django_tables2 import SingleTableMixin
+from django_tables2 import SingleTableView
 from rest_framework.parsers import FormParser
 from rest_framework.views import APIView
 
-from core.lib.opensearch import query_results
+from core.db.storage import db
+from core.lib.context import construct_query
 from core.lib.threshold import (
     annotate_num_chans,
     annotate_num_users,
     get_chans,
     get_users,
 )
+from core.views import helpers
 from core.views.ui.tables import DrilldownTable
 
-
-class DrilldownTableView(View, SingleTableMixin):
-    table_class = DrilldownTable
-    template_name = "ui/drilldown/table_results.html"
-    paginate_by = 5
-
-    def post(self, request):
-        context = query_results(request)
-        table = DrilldownTable(context["results"])
-        context["table"] = table
-        del context["results"]
-        if "message" in context:
-            return render(request, self.template_name, context)
-
-        if self.request.htmx:
-            template_name = "ui/drilldown/table_results.html"
-        else:
-            template_name = "ui/drilldown/table_results_partial.html"
-
-        if context:
-            return render(request, template_name, context)
-        else:
-            return HttpResponse("No results")
+# from copy import deepcopy
 
 
 def parse_dates(dates):
@@ -72,16 +53,86 @@ def create_tags(query):
     spl = query.split("AND")
     spl = [x.strip() for x in spl if ":" in x]
     spl = [x.replace('"', "") for x in spl]
-    tags = [f"{tag}: {elem}" for tag, elem in [x.split(":") for x in spl]]
+    tags = [f"{tag}: {elem}" for tag, elem in [x.split(":")[:2] for x in spl]]
     return tags
 
 
-def drilldown_search(request):
-    template_name = "ui/drilldown/results.html"
+def parse_tags(tags_pre):
+    """
+    Parse the tags from the variable tags_pre.
+    """
+    tags = {}
+    tags_spl = tags_pre.split(",")
+    if tags_spl:
+        for tag in tags_spl:
+            tag = tag.split(": ")
+            if len(tag) == 2:
+                key, val = tag
+                tags[key] = val
+    return tags
+
+
+def make_table(context):
+    table = DrilldownTable(context["object_list"])
+    context["table"] = table
+    # del context["results"]
+    return context
+
+
+def make_graph(results):
+    graph = []
+    for index, item in enumerate(results):
+        date = str(index)
+        graph.append(
+            {
+                "text": item.get("words_noun", None)
+                or item.get("msg", None)
+                or item.get("id"),
+                "nick": item.get("nick", None),
+                "channel": item.get("channel", None),
+                "net": item.get("net", None),
+                "value": item.get("sentiment", None) or None,
+                "date": date,
+            }
+        )
+    return ujson.dumps(graph)
+
+
+def drilldown_search(request, return_context=False, template=None):
+    extra_params = {}
+
+    if not template:
+        template_name = "widgets/table_results.html"
+    else:
+        template_name = template
+    if request.user.is_anonymous:
+        sizes = settings.MAIN_SIZES_ANON
+    else:
+        sizes = settings.MAIN_SIZES
+
     if request.GET:
+        if not request.htmx:
+            template_name = "ui/drilldown/drilldown.html"
         query_params = request.GET.dict()
     elif request.POST:
         query_params = request.POST.dict()
+    else:
+        template_name = "ui/drilldown/drilldown.html"
+        params_with_defaults = {}
+        helpers.add_defaults(params_with_defaults)
+        context = {"sizes": sizes, "unique": "results", "params": params_with_defaults}
+        return render(request, template_name, context)
+
+    tmp_post = request.POST.dict()
+    tmp_get = request.GET.dict()
+    tmp_post = {k: v for k, v in tmp_post.items() if v and not v == "None"}
+    tmp_get = {k: v for k, v in tmp_get.items() if v and not v == "None"}
+    query_params.update(tmp_post)
+    query_params.update(tmp_get)
+
+    # URI we're passing to the template for linking
+    if "csrfmiddlewaretoken" in query_params:
+        del query_params["csrfmiddlewaretoken"]
 
     # Parse the dates
     if "dates" in query_params:
@@ -95,41 +146,125 @@ def drilldown_search(request):
             query_params["from_time"] = dates["from_time"]
             query_params["to_time"] = dates["to_time"]
 
-    if request.GET:
-        context = query_results(request, query_params)
-    elif request.POST:
-        context = query_results(request, query_params)
+    if "query" in query_params:
+        # Remove null values
+        if query_params["query"] == "":
+            del query_params["query"]
+            # Turn the query into tags for populating the taglist
+            # tags = create_tags(query_params["query"])
+            # context["tags"] = tags
+    # else:
+    #     context = {"object_list": []}
 
-    # Turn the query into tags for populating the taglist
-    tags = create_tags(query_params["query"])
-    context["tags"] = tags
+    if "tags" in query_params:
+        if query_params["tags"] == "":
+            del query_params["tags"]
+        else:
+            tags = parse_tags(query_params["tags"])
+            extra_params["tags"] = tags
 
-    context["params"] = query_params
+    context = db.query_results(request, query_params, **extra_params)
+    context["unique"] = "results"
+
+    # Valid sizes
+    context["sizes"] = sizes
+
+    # Add any default parameters to the context
+    params_with_defaults = dict(query_params)
+    helpers.add_defaults(params_with_defaults)
+    context["params"] = params_with_defaults
+
+    helpers.remove_defaults(query_params)
+    url_params = urllib.parse.urlencode(query_params)
+    context["client_uri"] = url_params
     if "message" in context:
-        return render(request, template_name, context)
-
-    context["data"] = json.dumps(
-        [
-            {
-                "text": item.get("msg", None) or item.get("id"),
-                "nick": item.get("nick", None),
-                "value": item.get("sentiment", None) or None,
-                "date": item.get("ts"),
-            }
-            for item in context["results"]
-        ]
-    )
-    if context:
+        if return_context:
+            return context
         response = render(request, template_name, context)
         if request.GET:
-            return context
+            if request.htmx:
+                response["HX-Push"] = reverse("home") + "?" + url_params
         elif request.POST:
-            del query_params["csrfmiddlewaretoken"]
-            url_params = urllib.parse.urlencode(query_params)
             response["HX-Push"] = reverse("home") + "?" + url_params
         return response
-    else:
-        return HttpResponse("No results")
+
+    # Create data for chart.js sentiment graph
+    graph = make_graph(context["object_list"])
+    context["data"] = graph
+
+    context = make_table(context)
+
+    # URI we're passing to the template for linking, table fields removed
+    table_fields = ["page", "sort"]
+    clean_params = {k: v for k, v in query_params.items() if k not in table_fields}
+    clean_url_params = urllib.parse.urlencode(clean_params)
+    context["uri"] = clean_url_params
+
+    # unique = str(uuid.uuid4())[:8]
+    if return_context:
+        return context
+
+    response = render(request, template_name, context)
+    if request.GET:
+        if request.htmx:
+            response["HX-Push"] = reverse("home") + "?" + url_params
+    elif request.POST:
+        response["HX-Push"] = reverse("home") + "?" + url_params
+
+    return response
+
+
+class DrilldownTableView(SingleTableView):
+    table_class = DrilldownTable
+    template_name = "widgets/table_results.html"
+    paginate_by = settings.DRILLDOWN_RESULTS_PER_PAGE
+
+    def get_queryset(self, request, **kwargs):
+        context = drilldown_search(request, return_context=True)
+        # Save the context as we will need to merge other attributes later
+        self.context = context
+
+        if "object_list" in context:
+            return context["object_list"]
+        else:
+            return []
+
+    def get(self, request, *args, **kwargs):
+        self.object_list = self.get_queryset(request)
+        show = []
+        show = set().union(*(d.keys() for d in self.object_list))
+        allow_empty = self.get_allow_empty()
+
+        if not allow_empty:
+            # When pagination is enabled and object_list is a queryset,
+            # it's better to do a cheap query than to load the unpaginated
+            # queryset in memory.
+            if self.get_paginate_by(self.object_list) is not None and hasattr(
+                self.object_list, "exists"
+            ):
+                is_empty = not self.object_list.exists()  # noqa
+            else:
+                is_empty = not self.object_list  # noqa
+        context = self.get_context_data()
+        if isinstance(self.context, HttpResponse):
+            return self.context
+
+        for k, v in self.context.items():
+            if k not in context:
+                context[k] = v
+        context["show"] = show
+
+        if request.method == "GET":
+            if not request.htmx:
+                self.template_name = "ui/drilldown/drilldown.html"
+        response = self.render_to_response(context)
+        # if not request.method == "GET":
+        if "client_uri" in context:
+            response["HX-Push"] = reverse("home") + "?" + context["client_uri"]
+        return response
+
+    def post(self, request, *args, **kwargs):
+        return self.get(request, *args, **kwargs)
 
 
 class Drilldown(View):
@@ -137,27 +272,162 @@ class Drilldown(View):
     plan_name = "drilldown"
 
     def get(self, request):
-        if request.user.is_anonymous:
-            sizes = settings.OPENSEARCH_MAIN_SIZES_ANON
-        else:
-            sizes = settings.OPENSEARCH_MAIN_SIZES
-
-        context = {}
-        if request.GET:
-            context = drilldown_search(request)
-        context["sizes"] = sizes
-        return render(request, self.template_name, context)
+        return drilldown_search(request)
 
     def post(self, request):
         return drilldown_search(request)
 
 
+class DrilldownContextModal(APIView):
+    parser_classes = [FormParser]
+    plan_name = "drilldown"
+    template_name = "modals/context.html"
+
+    def post(self, request):
+        if request.resolver_match.url_name == "modal_context_table":
+            self.template_name = "partials/context_table.html"
+
+        size = 15
+        nicks_sensitive = None
+        query = False
+        # Create the query params from the POST arguments
+        mandatory = [
+            "net",
+            "channel",
+            "num",
+            "source",
+            "index",
+            "nick",
+            "type",
+            "mtype",
+        ]
+        invalid = [None, False, "—", "None"]
+
+        query_params = {k: v for k, v in request.data.items() if v}
+        for key in query_params:
+            if query_params[key] in invalid:
+                query_params[key] = None
+        for key in mandatory:
+            if key not in query_params:
+                query_params[key] = None
+
+        # Lookup the hash values but don't disclose them to the user
+        # if settings.HASHING:
+        #     if query_params["source"] not in settings.SAFE_SOURCES:
+        #         SAFE_PARAMS = deepcopy(query_params)
+        #         hash_lookup(request.user, SAFE_PARAMS)
+        #     else:
+        #         SAFE_PARAMS = deepcopy(query_params)
+        # else:
+        #     SAFE_PARAMS = query_params
+
+        type = None
+        if request.user.is_superuser:
+            if "type" in query_params:
+                type = query_params["type"]
+                if type == "znc":
+                    query_params["channel"] = "*status"
+                    # SAFE_PARAMS["channel"] = "*status"
+
+            if type in ["query", "notice"]:
+                nicks_sensitive = [
+                    query_params["channel"],
+                    query_params["nick"],
+                ]  # UNSAFE
+                # nicks = [query_params["channel"], query_params["nick"]]
+                query = True
+
+            if (
+                query_params["index"] == "internal"
+                and query_params["mtype"] == "msg"
+                and not type == "query"
+            ):
+                query_params["index"] = "main"
+                # SAFE_PARAMS["index"] = "main"
+
+            if query_params["type"] in ["znc", "auth"]:
+                query = True
+
+        if not request.user.is_superuser:
+            query_params["index"] = "main"
+            # SAFE_PARAMS["index"] = "main"
+
+        query_params["sorting"] = "desc"
+        # SAFE_PARAMS["sorting"] = "desc"
+        # query_params["size"] = size
+
+        annotate = False
+        if query_params["source"] == "irc":
+            if query_params["type"] not in ["znc", "auth"]:
+                annotate = True
+        # Create the query with the context helper
+        if query_params["num"].isdigit():
+            query_params["num"] = int(query_params["num"])
+        search_query = construct_query(
+            query_params["index"],
+            query_params["net"],
+            query_params["channel"],
+            query_params["source"],
+            query_params["num"],
+            size,
+            type=type,
+            nicks=nicks_sensitive,
+        )
+        results = db.query_results(
+            request,
+            query_params,
+            size=size,
+            annotate=annotate,
+            custom_query=search_query,
+            reverse=True,
+            dedup_fields=["net", "type", "msg"],
+        )
+        if "message" in results:
+            return render(request, self.template_name, results)
+
+        # if settings.HASHING:  # we probably want to see the tokens
+        #     if query_params["source"] not in settings.SAFE_SOURCES:
+        #         if not request.user.has_perm("core.bypass_hashing"):
+        #             for index, item in enumerate(results["object_list"]):
+        #                 if "tokens" in item:
+        #                     results["object_list"][index]["msg"] = results[
+        #                         "object_list"
+        #                     ][index].pop("tokens")
+        #                     # item["msg"] = item.pop("tokens")
+
+        # Make the time nicer
+        # for index, item in enumerate(results["object_list"]):
+        #    results["object_list"][index]["time"] = item["time"]+"SSS"
+        unique = str(uuid.uuid4())[:8]
+        context = {
+            "net": query_params["net"],
+            "channel": query_params["channel"],
+            "source": query_params["source"],
+            "ts": f"{query_params['date']} {query_params['time']}",
+            "object_list": results["object_list"],
+            "time": query_params["time"],
+            "date": query_params["date"],
+            "index": query_params["index"],
+            "num": query_params["num"],
+            "type": query_params["type"],
+            "mtype": query_params["mtype"],
+            "nick": query_params["nick"],
+            "params": query_params,
+            "unique": unique,
+        }
+        if request.user.is_superuser:
+            if query:
+                context["query"] = True
+
+        return render(request, self.template_name, context)
+
+
 class ThresholdInfoModal(APIView):
     parser_classes = [FormParser]
     plan_name = "drilldown"
     template_name = "modals/drilldown.html"
 
-    def post(self, request):
+    def post(self, request, type=None):
         # if not request.user.has_plan(self.plan_name):
         #    return JsonResponse({"success": False})
         if "net" not in request.data:
@@ -166,21 +436,56 @@ class ThresholdInfoModal(APIView):
             return JsonResponse({"success": False})
         if "channel" not in request.data:
             return JsonResponse({"success": False})
+        if type == "window":
+            self.template_name = "windows/drilldown.html"
+        elif type == "widget":
+            self.template_name = "widgets/drilldown.html"
+
         net = request.data["net"]
         nick = request.data["nick"]
         channel = request.data["channel"]
+
+        # SAFE BLOCK #
+        # Lookup the hash values but don't disclose them to the user
+        # if settings.HASHING:
+        #     SAFE_PARAMS = request.data.dict()
+        #     hash_lookup(request.user, SAFE_PARAMS)
+
         channels = get_chans(net, [nick])
-        users = get_users(net, [channel])
+        print("CHANNELS", channels)
+        users = get_users(net, [nick])
+        print("USERS", users)
         num_users = annotate_num_users(net, channels)
+        print("NUM_USERS", num_users)
         num_chans = annotate_num_chans(net, users)
+        print("NUM_CHANS", num_chans)
         if channels:
             inter_users = get_users(net, channels)
         else:
             inter_users = []
+        print("INTER_USERS", inter_users)
         if users:
             inter_chans = get_chans(net, users)
         else:
             inter_chans = []
+        print("INTER_CHANS", inter_chans)
+        # if settings.HASHING:
+        #     hash_list(request.user, inter_chans)
+        #     hash_list(request.user, inter_users)
+
+        #     hash_list(request.user, num_chans, hash_keys=True)
+        #     hash_list(request.user, num_users, hash_keys=True)
+
+        #     hash_list(request.user, channels)
+        #     hash_list(request.user, users)
+
+        # if settings.RANDOMISATION:
+        #     randomise_list(request.user, num_chans)
+        #     randomise_list(request.user, num_users)
+
+        # SAFE BLOCK END #
+
+        unique = str(uuid.uuid4())[:8]
         context = {
             "net": net,
             "nick": nick,
@@ -191,5 +496,7 @@ class ThresholdInfoModal(APIView):
             "inter_users": inter_users,
             "num_users": num_users,
             "num_chans": num_chans,
+            "unique": unique,
         }
+        print("CON", context)
         return render(request, self.template_name, context)

core/views/ui/insights.py

@@ -1,15 +1,15 @@
 from ast import literal_eval
 
-from django.contrib.auth.mixins import LoginRequiredMixin
+from django.contrib.auth.mixins import LoginRequiredMixin, PermissionRequiredMixin
 from django.http import HttpResponse, HttpResponseForbidden, JsonResponse
 from django.shortcuts import render
 from django.views import View
 from rest_framework.parsers import FormParser
 from rest_framework.views import APIView
 
+from core.db.druid import query_single_result
 from core.lib.meta import get_meta
 from core.lib.nicktrace import get_nicks
-from core.lib.opensearch import query_single_result
 from core.lib.threshold import (
     annotate_num_chans,
     annotate_num_users,
@@ -19,38 +19,35 @@ from core.lib.threshold import (
 )
 
 
-class Insights(LoginRequiredMixin, View):
+class Insights(LoginRequiredMixin, PermissionRequiredMixin, View):
     template_name = "ui/insights/insights.html"
-    plan_name = "drilldown"
+    permission_required = "use_insights"
 
     def get(self, request):
-        if not request.user.has_plan(self.plan_name):
-            return render(request, "denied.html")
         return render(request, self.template_name)
 
 
-class InsightsSearch(LoginRequiredMixin, View):
+class InsightsSearch(LoginRequiredMixin, PermissionRequiredMixin, View):
     # parser_classes = [JSONParser]
     template_name = "ui/insights/info.html"
-    plan_name = "drilldown"
+    permission_required = "use_insights"
 
     def post(self, request):
-        if not request.user.has_plan(self.plan_name):
-            return HttpResponseForbidden()
-        results, context = query_single_result(request)
+        query_params = request.POST.dict()
+        if "query_full" in query_params:
+            query_params["query_full"] = "nick: " + query_params["query_full"]
+        context = query_single_result(request, query_params)
         if not context:
             return HttpResponseForbidden()
         return render(request, self.template_name, context)
 
 
-class InsightsChannels(LoginRequiredMixin, APIView):
+class InsightsChannels(LoginRequiredMixin, PermissionRequiredMixin, APIView):
     parser_classes = [FormParser]
     template_name = "ui/insights/channels.html"
-    plan_name = "drilldown"
+    permission_required = "use_insights"
 
     def post(self, request):
-        if not request.user.has_plan(self.plan_name):
-            return HttpResponseForbidden()
         if "net" not in request.data:
             return HttpResponse("No net")
         if "nick" not in request.data:
@@ -65,14 +62,12 @@ class InsightsChannels(LoginRequiredMixin, APIView):
         return render(request, self.template_name, context)
 
 
-class InsightsNicks(LoginRequiredMixin, APIView):
+class InsightsNicks(LoginRequiredMixin, PermissionRequiredMixin, APIView):
     parser_classes = [FormParser]
     template_name = "ui/insights/nicks.html"
-    plan_name = "drilldown"
+    permission_required = "use_insights"
 
     def post(self, request):
-        if not request.user.has_plan(self.plan_name):
-            return HttpResponseForbidden()
         if "net" not in request.data:
             return HttpResponse("No net")
         if "nick" not in request.data:
@@ -91,14 +86,12 @@ class InsightsNicks(LoginRequiredMixin, APIView):
         return render(request, self.template_name, context)
 
 
-class InsightsMeta(LoginRequiredMixin, APIView):
+class InsightsMeta(LoginRequiredMixin, PermissionRequiredMixin, APIView):
     parser_classes = [FormParser]
     template_name = "ui/insights/meta.html"
-    plan_name = "drilldown"
+    permission_required = "use_insights"
 
     def post(self, request):
-        if not request.user.has_plan(self.plan_name):
-            return HttpResponseForbidden()
         if "net" not in request.data:
             return HttpResponse("No net")
         if "nicks" not in request.data:
@@ -133,14 +126,12 @@ class InsightsMeta(LoginRequiredMixin, APIView):
         return render(request, self.template_name, context)
 
 
-class InsightsInfoModal(LoginRequiredMixin, APIView):
+class InsightsInfoModal(LoginRequiredMixin, PermissionRequiredMixin, APIView):
     parser_classes = [FormParser]
-    plan_name = "drilldown"
     template_name = "modals/drilldown.html"
+    permission_required = "use_insights"
 
     def post(self, request):
-        if not request.user.has_plan(self.plan_name):
-            return JsonResponse({"success": False})
         if "net" not in request.data:
             return JsonResponse({"success": False})
         if "nick" not in request.data:

core/views/ui/tables.py

@@ -1,4 +1,27 @@
+from django.conf import settings
 from django_tables2 import Column, Table
+from django_tables2.columns.base import BoundColumn
+
+# Make the table column headings lowercase
+orig_Column_header = BoundColumn.header
+
+
+@property
+def format_header(self):
+    header = orig_Column_header.__get__(self)
+    header = header.lower()
+    header = header.title()
+    if header != "Ident":
+        header = header.replace("Id", "ID")
+        header = header.replace("id", "ID")
+    if header == "Ts":
+        header = "TS"
+    header = header.replace("Nsfw", "NSFW")
+
+    return header
+
+
+BoundColumn.header = format_header
 
 
 class DrilldownTable(Table):
@@ -15,6 +38,8 @@ class DrilldownTable(Table):
     num = Column()
     src = Column()
     ts = Column()
+    date = Column()
+    time = Column()
     type = Column()
     bot = Column()
     channel = Column()
@@ -31,4 +56,27 @@ class DrilldownTable(Table):
     sentiment = Column()
     status = Column()
     user = Column()
-    version_sentiment = Column()
+    # version_sentiment = Column()
+    exemption = Column()
+    num_chans = Column()
+    num_users = Column()
+    online = Column()
+    mtype = Column()
+    realname = Column()
+    server = Column()
+    mtype = Column()
+    # tokens = Column()
+    lang_code = Column()
+    lang_name = Column()
+    words_noun = Column()
+    words_adj = Column()
+    words_verb = Column()
+    words_adv = Column()
+    hidden = Column()
+    filename = Column()
+    file_md5 = Column()
+    file_ext = Column()
+    file_size = Column()
+
+    template_name = "ui/drilldown/table_results.html"
+    paginate_by = settings.DRILLDOWN_RESULTS_PER_PAGE

docker-compose.yml

@@ -12,6 +12,26 @@ services:
       - "${NEPTUNE_PORT}:8000"
     env_file:
       - .env
+    volumes_from:
+        - tmp
+    depends_on:
+      redis:
+        condition: service_healthy
+      migration:
+        condition: service_started
+
+  migration:
+    image: pathogen/neptune:latest
+    command: sh -c '. /venv/bin/activate && python manage.py migrate --noinput'
+    volumes:
+      - ${PORTAINER_GIT_DIR}:/code
+      - ${NEPTUNE_LOCAL_SETTINGS}:/code/app/local_settings.py
+      - ${NEPTUNE_DATABASE_FILE}:/code/db.sqlite3
+    volumes_from:
+      - tmp
+    depends_on:
+      redis:
+        condition: service_healthy
 
   # pyroscope:
   #   image: pyroscope/pyroscope
@@ -22,6 +42,25 @@ services:
   #   command:
   #     - 'server'
 
+  tmp:
+    image: busybox
+    command: chmod -R 777 /var/run/redis
+    volumes:
+      - /var/run/redis
+
+  redis:
+    image: redis
+    command: redis-server /etc/redis.conf
+    volumes:
+      - ${PORTAINER_GIT_DIR}/docker/redis.conf:/etc/redis.conf
+    volumes_from:
+      - tmp
+    healthcheck:
+      test: "redis-cli -s /var/run/redis/redis.sock ping"
+      interval: 2s
+      timeout: 2s
+      retries: 15
+
 networks:
    default:
       external:

docker/docker-compose.prod.yml

@@ -13,6 +13,46 @@ services:
       - "${NEPTUNE_PORT}:8000" # uwsgi socket
     env_file:
       - ../stack.env
+    volumes_from:
+      - tmp
+    depends_on:
+      redis:
+        condition: service_healthy
+      migration:
+        condition: service_started
+
+  migration:
+    image: pathogen/neptune:latest
+    build: ./docker/prod
+    command: sh -c '. /venv/bin/activate && python manage.py migrate --noinput'
+    volumes:
+      - ${PORTAINER_GIT_DIR}:/code
+      - ${NEPTUNE_LOCAL_SETTINGS}:/code/app/local_settings.py
+      - ${NEPTUNE_DATABASE_FILE}:/code/db.sqlite3
+    volumes_from:
+      - tmp
+    depends_on:
+      redis:
+        condition: service_healthy
+
+  tmp:
+    image: busybox
+    command: chmod -R 777 /var/run/redis
+    volumes:
+      - /var/run/redis
+
+  redis:
+    image: redis
+    command: redis-server /etc/redis.conf
+    volumes:
+        - ${PORTAINER_GIT_DIR}/docker/redis.conf:/etc/redis.conf
+    volumes_from:
+        - tmp
+    healthcheck:
+      test: "redis-cli -s /var/run/redis/redis.sock ping"
+      interval: 2s
+      timeout: 2s
+      retries: 15
 
 networks:
    default:

docker/prod/requirements.prod.txt

@@ -2,10 +2,18 @@ wheel
 django
 django-crispy-forms
 crispy-bulma
-opensearch-py
+#opensearch-py
 stripe
 django-rest-framework
 numpy
 uwsgi
 django-tables2
 django-tables2-bulma-template
+django-htmx
+cryptography
+siphashc
+redis
+sortedcontainers
+django-debug-toolbar
+django-debug-toolbar-template-profiler
+orjson

docker/redis.conf

@@ -0,0 +1,2 @@
+unixsocket /var/run/redis/redis.sock
+unixsocketperm 777

docker/requirements.dev.txt

@@ -2,10 +2,17 @@ wheel
 django
 django-crispy-forms
 crispy-bulma
-opensearch-py
+#opensearch-py
 stripe
 django-rest-framework
 numpy
 django-tables2
 django-tables2-bulma-template
 django-htmx
+cryptography
+siphashc
+redis
+sortedcontainers
+django-debug-toolbar
+django-debug-toolbar-template-profiler
+orjson

pyproject.toml

@@ -11,7 +11,7 @@ exclude = '''
 )
 '''
 [tool.curlylint.rules]
-indent = 2
+#indent = 2
 html_has_lang = 'en-GB'
 # All role attributes must be valid.
 # See https://www.curlylint.org/docs/rules/aria_role.

requirements.txt

@@ -3,9 +3,17 @@ django
 pre-commit
 django-crispy-forms
 crispy-bulma
-opensearch-py
+#opensearch-py
 stripe
 django-rest-framework
 numpy
 django-tables2
 django-tables2-bulma-template
+django-htmx
+cryptography
+siphashc
+redis
+sortedcontainers
+django-debug-toolbar
+django-debug-toolbar-template-profiler
+orjson