Fix insights

app/urls.py

@@ -64,15 +64,14 @@ from core.views.ui.drilldown import (  # DrilldownTableView,; Drilldown,
     DrilldownTableView,
     ThresholdInfoModal,
 )
-
+from core.views.ui.insights import (
-# from core.views.ui.insights import (
+    Insights,
-#     Insights,
+    InsightsChannels,
-#     InsightsChannels,
+    InsightsInfoModal,
-#     InsightsInfoModal,
+    InsightsMeta,
-#     InsightsMeta,
+    InsightsNicks,
-#     InsightsNicks,
+    InsightsSearch,
-#     InsightsSearch,
+)
-# )
 
 urlpatterns = [
     path("__debug__/", include("debug_toolbar.urls")),
@@ -103,12 +102,32 @@ urlpatterns = [
     path("context/", DrilldownContextModal.as_view(), name="modal_context"),
     path("context_table/", DrilldownContextModal.as_view(), name="modal_context_table"),
     ##
-    # path("ui/insights/", Insights.as_view(), name="insights"),
+    path("ui/insights/index/<str:index>/", Insights.as_view(), name="insights"),
-    # path("ui/insights/search/", InsightsSearch.as_view(), name="search_insights"),
+    path(
-    # path("ui/insights/channels/", InsightsChannels.as_view(), name="chans_insights"),
+        "ui/insights/index/<str:index>/search/",
-    # path("ui/insights/nicks/", InsightsNicks.as_view(), name="nicks_insights"),
+        InsightsSearch.as_view(),
-    # path("ui/insights/meta/", InsightsMeta.as_view(), name="meta_insights"),
+        name="search_insights",
-    # path("ui/insights/modal/", InsightsInfoModal.as_view(), name="modal_insights"),
+    ),
+    path(
+        "ui/insights/index/<str:index>/channels/",
+        InsightsChannels.as_view(),
+        name="chans_insights",
+    ),
+    path(
+        "ui/insights/index/<str:index>/nicks/",
+        InsightsNicks.as_view(),
+        name="nicks_insights",
+    ),
+    path(
+        "ui/insights/index/<str:index>/meta/",
+        InsightsMeta.as_view(),
+        name="meta_insights",
+    ),
+    path(
+        "ui/insights/index/<str:index>/modal/",
+        InsightsInfoModal.as_view(),
+        name="modal_insights",
+    ),
     ##
     path(
         "manage/threshold/irc/overview/",

core/lib/meta.py

@@ -3,7 +3,7 @@ from math import ceil
 from django.conf import settings
 from numpy import array_split
 
-from core.db.elastic import client, run_main_query
+from core.db.storage import db
 
 
 def construct_query(net, nicks):
@@ -43,26 +43,13 @@ def get_meta(request, net, nicks, iter=True):
             break
         meta_tmp = []
         query = construct_query(net, nicks_chunked)
-        results = run_main_query(
+        results = db.query(
-            client,
             request.user,
             query,
-            custom_query=True,
+            index=settings.INDEX_META,
-            index=settings.ELASTICSEARCH_INDEX_META,
         )
-        if "hits" in results.keys():
+        if "object_list" in results.keys():
-            if "hits" in results["hits"]:
+            for element in results["object_list"]:
-                for item in results["hits"]["hits"]:
-                    element = item["_source"]
-                    element["id"] = item["_id"]
-
-                    # Split the timestamp into date and time
-                    ts = element["ts"]
-                    ts_spl = ts.split("T")
-                    date = ts_spl[0]
-                    time = ts_spl[1]
-                    element["date"] = date
-                    element["time"] = time
                 meta_tmp.append(element)
         for x in meta_tmp:
             if x not in meta:

core/lib/nicktrace.py

@@ -3,7 +3,7 @@ from math import ceil
 from django.conf import settings
 from numpy import array_split
 
-from core.lib.druid import client, run_main_query
+from core.db.storage import db
 
 
 def construct_query(net, nicks):
@@ -45,7 +45,7 @@ def get_nicks(request, net, nicks, iter=True):
         if len(nicks_chunked) == 0:
             break
         query = construct_query(net, nicks_chunked)
-        results = run_main_query(client, request.user, query, custom_query=True)
+        results = db.query(request.user, query)
         if "hits" in results.keys():
             if "hits" in results["hits"]:
                 for item in results["hits"]["hits"]:

core/models.py

@@ -1,6 +1,7 @@
 import logging
 
 import stripe
+from django.conf import settings
 from django.contrib.auth.models import AbstractUser
 from django.db import models
 from yaml import load
@@ -24,15 +25,13 @@ PRIORITY_CHOICES = (
 )
 
 INTERVAL_CHOICES = (
-    ("ondemand", "On demand"),
+    (0, "On demand"),
-    ("minute", "Every minute"),
+    (60, "Every minute"),
-    ("15m", "Every 15 minutes"),
+    (900, "Every 15 minutes"),
-    ("30m", "Every 30 minutes"),
+    (1800, "Every 30 minutes"),
-    ("hour", "Every hour"),
+    (3600, "Every hour"),
-    ("4h", "Every 4 hours"),
+    (14400, "Every 4 hours"),
-    ("day", "Every day"),
+    (86400, "Every day"),
-    ("week", "Every week"),
-    ("month", "Every month"),
 )
 
 
@@ -90,6 +89,19 @@ class User(AbstractUser):
     def get_notification_settings(self):
         return NotificationSettings.objects.get_or_create(user=self)[0]
 
+    @property
+    def allowed_indices(self):
+        indices = [settings.INDEX_MAIN]
+        if self.has_perm("core.index_meta"):
+            indices.append(settings.INDEX_META)
+        if self.has_perm("core.index_internal"):
+            indices.append(settings.INDEX_INT)
+        if self.has_perm("core.index_restricted"):
+            if self.has_perm("core.restricted_sources"):
+                indices.append(settings.INDEX_RESTRICTED)
+
+        return indices
+
 
 class Session(models.Model):
     user = models.ForeignKey(User, on_delete=models.CASCADE)
@@ -137,16 +149,10 @@ class ContentBlock(models.Model):
 class Perms(models.Model):
     class Meta:
         permissions = (
-            ("bypass_hashing", "Can bypass field hashing"),  #
-            ("bypass_blacklist", "Can bypass the blacklist"),  #
-            ("bypass_encryption", "Can bypass field encryption"),  #
-            ("bypass_obfuscation", "Can bypass field obfuscation"),  #
-            ("bypass_delay", "Can bypass data delay"),  #
-            ("bypass_randomisation", "Can bypass data randomisation"),  #
             ("post_irc", "Can post to IRC"),
             ("post_discord", "Can post to Discord"),
-            ("query_search", "Can search with query strings"),  #
             ("use_insights", "Can use the Insights page"),
+            ("use_rules", "Can use the Rules page"),
             ("index_internal", "Can use the internal index"),
             ("index_meta", "Can use the meta index"),
             ("index_restricted", "Can use the restricted index"),
@@ -159,9 +165,7 @@ class NotificationRule(models.Model):
     name = models.CharField(max_length=255)
     priority = models.IntegerField(choices=PRIORITY_CHOICES, default=1)
     topic = models.CharField(max_length=255, null=True, blank=True)
-    interval = models.CharField(
+    interval = models.IntegerField(choices=INTERVAL_CHOICES, default=0)
-        choices=INTERVAL_CHOICES, max_length=255, default="ondemand"
-    )
     window = models.CharField(max_length=255, null=True, blank=True)
     enabled = models.BooleanField(default=True)
     data = models.TextField()

core/templates/base.html

@@ -286,9 +286,21 @@
           {% endif %}
 
           {% if perms.core.use_insights %}
-            <a class="navbar-item" href="{# url 'insights' #}">
+            <div class="navbar-item has-dropdown is-hoverable">
+              <a class="navbar-link">
                 Insights
               </a>
+
+              <div class="navbar-dropdown">
+                {% for index in user.allowed_indices %}
+                  {% if index != "meta" %}
+                    <a class="navbar-item" href="{% url 'insights' index=index %}">
+                      {{ index }}
+                    </a>
+                  {% endif %}
+                {% endfor %}
+              </div>
+            </div>
           {% endif %}
           <a class="navbar-item add-button">
             Install

core/templates/partials/rule-list.html

@@ -11,6 +11,8 @@
     <th>id</th>
     <th>user</th>
     <th>name</th>
+    <th>interval</th>
+    <th>window</th>
     <th>priority</th>
     <th>topic</th>
     <th>enabled</th>
@@ -22,6 +24,8 @@
       <td>{{ item.id }}</td>
       <td>{{ item.user }}</td>
       <td>{{ item.name }}</td>
+      <td>{{ item.interval }}s</td>
+      <td>{{ item.window|default_if_none:"—" }}</td>
       <td>{{ item.priority }}</td>
       <td>{{ item.topic|default_if_none:"—" }}</td>
       <td>

core/templates/ui/insights/info.html

@@ -4,7 +4,7 @@
   style="display: none;"
   hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
   hx-vals='{"net": "{{ item.net }}", "nick": "{{ item.nick }}"}'
-  hx-post="{% url 'chans_insights' %}"
+  hx-post="{% url 'chans_insights' index=index %}"
   hx-trigger="load"
   hx-target="#channels"
   hx-swap="outerHTML">
@@ -13,7 +13,7 @@
   style="display: none;"
   hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
   hx-vals='{"net": "{{ item.net }}", "nick": "{{ item.nick }}"}'
-  hx-post="{% url 'nicks_insights' %}"
+  hx-post="{% url 'nicks_insights' index=index %}"
   hx-trigger="load"
   hx-target="#nicks"
   hx-swap="outerHTML">
@@ -81,7 +81,7 @@
                 {% if item.src == 'irc' %}
                   <button
                     hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
-                    hx-post="{% url 'modal_insights' %}"
+                    hx-post="{% url 'modal_insights' index=index %}"
                     hx-vals='{"net": "{{ item.net }}", "nick": "{{ item.nick }}", "channel": "{{ item.channel }}"}'
                     hx-target="#modals-here"
                     hx-trigger="click"

core/templates/ui/insights/insights.html

@@ -2,39 +2,7 @@
 {% load static %}
 {% block content %}
   {% include 'partials/notify.html' %}
-  <script>
+  <script src="{% static 'tabs.js' %}"></script>
-    // tabbed browsing for the modal
-    function initTabs() {
-      TABS.forEach((tab) => {
-        tab.addEventListener('click', (e) => {
-          let selected = tab.getAttribute('data-tab');
-          updateActiveTab(tab);
-          updateActiveContent(selected);
-        })
-      })
-    }
-
-    function updateActiveTab(selected) {
-      TABS.forEach((tab) => {
-        if (tab && tab.classList.contains(ACTIVE_CLASS)) {
-          tab.classList.remove(ACTIVE_CLASS);
-        }
-      });
-      selected.classList.add(ACTIVE_CLASS);
-    }
-
-    function updateActiveContent(selected) {
-      CONTENT.forEach((item) => {
-        if (item && item.classList.contains(ACTIVE_CLASS)) {
-          item.classList.remove(ACTIVE_CLASS);
-        }
-        let data = item.getAttribute('data-content');
-        if (data === selected) {
-          item.classList.add(ACTIVE_CLASS);
-        }
-      });
-    }
-  </script>
   <style>
     .icon { border-bottom: 0px !important;}
   </style>
@@ -47,7 +15,7 @@
               {% csrf_token %}
               <div class="field has-addons">
                 <div class="control is-expanded has-icons-left">
-                  <input id="query_full" name="query_full" class="input" type="text" placeholder="nickname">
+                  <input id="query_full" name="query" class="input" type="text" placeholder="nickname">
                   <span class="icon is-small is-left">
                     <i class="fas fa-magnifying-glass"></i>
                   </span>
@@ -55,7 +23,7 @@
                 <div class="control">
                   <button
                     class="button is-info is-fullwidth"
-                    hx-post="{% url 'search_insights' %}"
+                    hx-post="{% url 'search_insights' index=index %}"
                     hx-trigger="click"
                     hx-target="#info"
                     hx-swap="outerHTML">

core/templates/ui/insights/nicks.html

@@ -3,7 +3,7 @@
   style="display: none;"
   hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
   hx-vals='{"net": "{{ net }}", "nicks": "{{ nicks }}"}'
-  hx-post="{% url 'meta_insights' %}"
+  hx-post="{% url 'meta_insights' index=index %}"
   hx-trigger="load"
   hx-target="#meta"
   hx-swap="outerHTML">

core/views/ui/insights.py

@@ -7,7 +7,7 @@ from django.views import View
 from rest_framework.parsers import FormParser
 from rest_framework.views import APIView
 
-from core.db.druid import query_single_result
+from core.db.storage import db
 from core.lib.meta import get_meta
 from core.lib.nicktrace import get_nicks
 from core.lib.threshold import (
@@ -23,8 +23,9 @@ class Insights(LoginRequiredMixin, PermissionRequiredMixin, View):
     template_name = "ui/insights/insights.html"
     permission_required = "use_insights"
 
-    def get(self, request):
+    def get(self, request, index):
-        return render(request, self.template_name)
+        context = {"index": index}
+        return render(request, self.template_name, context)
 
 
 class InsightsSearch(LoginRequiredMixin, PermissionRequiredMixin, View):
@@ -32,13 +33,16 @@ class InsightsSearch(LoginRequiredMixin, PermissionRequiredMixin, View):
     template_name = "ui/insights/info.html"
     permission_required = "use_insights"
 
-    def post(self, request):
+    def post(self, request, index):
         query_params = request.POST.dict()
-        if "query_full" in query_params:
+        if "query" in query_params:
-            query_params["query_full"] = "nick: " + query_params["query_full"]
+            query_params["query"] = "nick: " + query_params["query"]
-        context = query_single_result(request, query_params)
+        query_params["source"] = "all"
+        query_params["index"] = index
+        context = db.query_single_result(request, query_params)
         if not context:
             return HttpResponseForbidden()
+        context["index"] = index
         return render(request, self.template_name, context)
 
 
@@ -47,7 +51,7 @@ class InsightsChannels(LoginRequiredMixin, PermissionRequiredMixin, APIView):
     template_name = "ui/insights/channels.html"
     permission_required = "use_insights"
 
-    def post(self, request):
+    def post(self, request, index):
         if "net" not in request.data:
             return HttpResponse("No net")
         if "nick" not in request.data:
@@ -58,7 +62,13 @@ class InsightsChannels(LoginRequiredMixin, PermissionRequiredMixin, APIView):
         num_users = annotate_num_users(net, chans)
         if not chans:
             return HttpResponseForbidden()
-        context = {"net": net, "nick": nick, "chans": chans, "num_users": num_users}
+        context = {
+            "net": net,
+            "nick": nick,
+            "chans": chans,
+            "num_users": num_users,
+            "index": index,
+        }
         return render(request, self.template_name, context)
 
 
@@ -67,7 +77,7 @@ class InsightsNicks(LoginRequiredMixin, PermissionRequiredMixin, APIView):
     template_name = "ui/insights/nicks.html"
     permission_required = "use_insights"
 
-    def post(self, request):
+    def post(self, request, index):
         if "net" not in request.data:
             return HttpResponse("No net")
         if "nick" not in request.data:
@@ -82,7 +92,13 @@ class InsightsNicks(LoginRequiredMixin, PermissionRequiredMixin, APIView):
         online = annotate_online(net, nicks)
         if not nicks:
             return HttpResponseForbidden()
-        context = {"net": net, "nick": nick, "nicks": nicks, "online": online}
+        context = {
+            "net": net,
+            "nick": nick,
+            "nicks": nicks,
+            "online": online,
+            "index": index,
+        }
         return render(request, self.template_name, context)
 
 
@@ -91,7 +107,7 @@ class InsightsMeta(LoginRequiredMixin, PermissionRequiredMixin, APIView):
     template_name = "ui/insights/meta.html"
     permission_required = "use_insights"
 
-    def post(self, request):
+    def post(self, request, index):
         if "net" not in request.data:
             return HttpResponse("No net")
         if "nicks" not in request.data:
@@ -99,6 +115,10 @@ class InsightsMeta(LoginRequiredMixin, PermissionRequiredMixin, APIView):
         net = request.data["net"]
         nicks = request.data["nicks"]
         nicks = literal_eval(nicks)
+
+        # Check the user has permissions to use the meta index
+        if not request.user.has_perm("core.index_meta"):
+            return HttpResponseForbidden()
         meta = get_meta(request, net, nicks)
         unique_values = {}
         # Create a map of unique values for each key for each nick
@@ -122,7 +142,7 @@ class InsightsMeta(LoginRequiredMixin, PermissionRequiredMixin, APIView):
                     meta_dedup[k].add(v)
                     unique_values[nick][k].remove(v)
 
-        context = {"net": net, "nicks": nicks, "meta": meta_dedup}
+        context = {"net": net, "nicks": nicks, "meta": meta_dedup, "index": index}
         return render(request, self.template_name, context)
 
 
@@ -131,7 +151,7 @@ class InsightsInfoModal(LoginRequiredMixin, PermissionRequiredMixin, APIView):
     template_name = "modals/drilldown.html"
     permission_required = "use_insights"
 
-    def post(self, request):
+    def post(self, request, index):
         if "net" not in request.data:
             return JsonResponse({"success": False})
         if "nick" not in request.data:
@@ -163,5 +183,6 @@ class InsightsInfoModal(LoginRequiredMixin, PermissionRequiredMixin, APIView):
             "inter_users": inter_users,
             "num_users": num_users,
             "num_chans": num_chans,
+            "index": index,
         }
         return render(request, self.template_name, context)