From 9ee9c7abde6dafa9a1dfd229cd4db242e7f97ed2 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sat, 14 Jan 2023 16:36:00 +0000 Subject: [PATCH] Fix insights --- app/urls.py | 49 ++++++++++++++++-------- core/lib/meta.py | 25 +++--------- core/lib/nicktrace.py | 4 +- core/models.py | 42 +++++++++++--------- core/templates/base.html | 18 +++++++-- core/templates/partials/rule-list.html | 4 ++ core/templates/ui/insights/info.html | 6 +-- core/templates/ui/insights/insights.html | 38 ++---------------- core/templates/ui/insights/nicks.html | 2 +- core/views/ui/insights.py | 49 +++++++++++++++++------- 10 files changed, 126 insertions(+), 111 deletions(-) diff --git a/app/urls.py b/app/urls.py index 4a6b3be..1e8b2fe 100644 --- a/app/urls.py +++ b/app/urls.py @@ -64,15 +64,14 @@ from core.views.ui.drilldown import ( # DrilldownTableView,; Drilldown, DrilldownTableView, ThresholdInfoModal, ) - -# from core.views.ui.insights import ( -# Insights, -# InsightsChannels, -# InsightsInfoModal, -# InsightsMeta, -# InsightsNicks, -# InsightsSearch, -# ) +from core.views.ui.insights import ( + Insights, + InsightsChannels, + InsightsInfoModal, + InsightsMeta, + InsightsNicks, + InsightsSearch, +) urlpatterns = [ path("__debug__/", include("debug_toolbar.urls")), @@ -103,12 +102,32 @@ urlpatterns = [ path("context/", DrilldownContextModal.as_view(), name="modal_context"), path("context_table/", DrilldownContextModal.as_view(), name="modal_context_table"), ## - # path("ui/insights/", Insights.as_view(), name="insights"), - # path("ui/insights/search/", InsightsSearch.as_view(), name="search_insights"), - # path("ui/insights/channels/", InsightsChannels.as_view(), name="chans_insights"), - # path("ui/insights/nicks/", InsightsNicks.as_view(), name="nicks_insights"), - # path("ui/insights/meta/", InsightsMeta.as_view(), name="meta_insights"), - # path("ui/insights/modal/", InsightsInfoModal.as_view(), name="modal_insights"), + path("ui/insights/index//", Insights.as_view(), name="insights"), + path( + "ui/insights/index//search/", + InsightsSearch.as_view(), + name="search_insights", + ), + path( + "ui/insights/index//channels/", + InsightsChannels.as_view(), + name="chans_insights", + ), + path( + "ui/insights/index//nicks/", + InsightsNicks.as_view(), + name="nicks_insights", + ), + path( + "ui/insights/index//meta/", + InsightsMeta.as_view(), + name="meta_insights", + ), + path( + "ui/insights/index//modal/", + InsightsInfoModal.as_view(), + name="modal_insights", + ), ## path( "manage/threshold/irc/overview/", diff --git a/core/lib/meta.py b/core/lib/meta.py index a5ee778..ae0a56f 100644 --- a/core/lib/meta.py +++ b/core/lib/meta.py @@ -3,7 +3,7 @@ from math import ceil from django.conf import settings from numpy import array_split -from core.db.elastic import client, run_main_query +from core.db.storage import db def construct_query(net, nicks): @@ -43,27 +43,14 @@ def get_meta(request, net, nicks, iter=True): break meta_tmp = [] query = construct_query(net, nicks_chunked) - results = run_main_query( - client, + results = db.query( request.user, query, - custom_query=True, - index=settings.ELASTICSEARCH_INDEX_META, + index=settings.INDEX_META, ) - if "hits" in results.keys(): - if "hits" in results["hits"]: - for item in results["hits"]["hits"]: - element = item["_source"] - element["id"] = item["_id"] - - # Split the timestamp into date and time - ts = element["ts"] - ts_spl = ts.split("T") - date = ts_spl[0] - time = ts_spl[1] - element["date"] = date - element["time"] = time - meta_tmp.append(element) + if "object_list" in results.keys(): + for element in results["object_list"]: + meta_tmp.append(element) for x in meta_tmp: if x not in meta: meta.append(x) diff --git a/core/lib/nicktrace.py b/core/lib/nicktrace.py index 41e4b2d..49f0005 100644 --- a/core/lib/nicktrace.py +++ b/core/lib/nicktrace.py @@ -3,7 +3,7 @@ from math import ceil from django.conf import settings from numpy import array_split -from core.lib.druid import client, run_main_query +from core.db.storage import db def construct_query(net, nicks): @@ -45,7 +45,7 @@ def get_nicks(request, net, nicks, iter=True): if len(nicks_chunked) == 0: break query = construct_query(net, nicks_chunked) - results = run_main_query(client, request.user, query, custom_query=True) + results = db.query(request.user, query) if "hits" in results.keys(): if "hits" in results["hits"]: for item in results["hits"]["hits"]: diff --git a/core/models.py b/core/models.py index 853937b..43c16ec 100644 --- a/core/models.py +++ b/core/models.py @@ -1,6 +1,7 @@ import logging import stripe +from django.conf import settings from django.contrib.auth.models import AbstractUser from django.db import models from yaml import load @@ -24,15 +25,13 @@ PRIORITY_CHOICES = ( ) INTERVAL_CHOICES = ( - ("ondemand", "On demand"), - ("minute", "Every minute"), - ("15m", "Every 15 minutes"), - ("30m", "Every 30 minutes"), - ("hour", "Every hour"), - ("4h", "Every 4 hours"), - ("day", "Every day"), - ("week", "Every week"), - ("month", "Every month"), + (0, "On demand"), + (60, "Every minute"), + (900, "Every 15 minutes"), + (1800, "Every 30 minutes"), + (3600, "Every hour"), + (14400, "Every 4 hours"), + (86400, "Every day"), ) @@ -90,6 +89,19 @@ class User(AbstractUser): def get_notification_settings(self): return NotificationSettings.objects.get_or_create(user=self)[0] + @property + def allowed_indices(self): + indices = [settings.INDEX_MAIN] + if self.has_perm("core.index_meta"): + indices.append(settings.INDEX_META) + if self.has_perm("core.index_internal"): + indices.append(settings.INDEX_INT) + if self.has_perm("core.index_restricted"): + if self.has_perm("core.restricted_sources"): + indices.append(settings.INDEX_RESTRICTED) + + return indices + class Session(models.Model): user = models.ForeignKey(User, on_delete=models.CASCADE) @@ -137,16 +149,10 @@ class ContentBlock(models.Model): class Perms(models.Model): class Meta: permissions = ( - ("bypass_hashing", "Can bypass field hashing"), # - ("bypass_blacklist", "Can bypass the blacklist"), # - ("bypass_encryption", "Can bypass field encryption"), # - ("bypass_obfuscation", "Can bypass field obfuscation"), # - ("bypass_delay", "Can bypass data delay"), # - ("bypass_randomisation", "Can bypass data randomisation"), # ("post_irc", "Can post to IRC"), ("post_discord", "Can post to Discord"), - ("query_search", "Can search with query strings"), # ("use_insights", "Can use the Insights page"), + ("use_rules", "Can use the Rules page"), ("index_internal", "Can use the internal index"), ("index_meta", "Can use the meta index"), ("index_restricted", "Can use the restricted index"), @@ -159,9 +165,7 @@ class NotificationRule(models.Model): name = models.CharField(max_length=255) priority = models.IntegerField(choices=PRIORITY_CHOICES, default=1) topic = models.CharField(max_length=255, null=True, blank=True) - interval = models.CharField( - choices=INTERVAL_CHOICES, max_length=255, default="ondemand" - ) + interval = models.IntegerField(choices=INTERVAL_CHOICES, default=0) window = models.CharField(max_length=255, null=True, blank=True) enabled = models.BooleanField(default=True) data = models.TextField() diff --git a/core/templates/base.html b/core/templates/base.html index eef348d..e5b6a83 100644 --- a/core/templates/base.html +++ b/core/templates/base.html @@ -286,9 +286,21 @@ {% endif %} {% if perms.core.use_insights %} - - Insights - +
+ + Insights + + +
+ {% for index in user.allowed_indices %} + {% if index != "meta" %} + + {{ index }} + + {% endif %} + {% endfor %} +
+
{% endif %} Install diff --git a/core/templates/partials/rule-list.html b/core/templates/partials/rule-list.html index 1f486cf..c854b00 100644 --- a/core/templates/partials/rule-list.html +++ b/core/templates/partials/rule-list.html @@ -11,6 +11,8 @@ id user name + interval + window priority topic enabled @@ -22,6 +24,8 @@ {{ item.id }} {{ item.user }} {{ item.name }} + {{ item.interval }}s + {{ item.window|default_if_none:"—" }} {{ item.priority }} {{ item.topic|default_if_none:"—" }} diff --git a/core/templates/ui/insights/info.html b/core/templates/ui/insights/info.html index 0282c39..1e6dce5 100644 --- a/core/templates/ui/insights/info.html +++ b/core/templates/ui/insights/info.html @@ -4,7 +4,7 @@ style="display: none;" hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}' hx-vals='{"net": "{{ item.net }}", "nick": "{{ item.nick }}"}' - hx-post="{% url 'chans_insights' %}" + hx-post="{% url 'chans_insights' index=index %}" hx-trigger="load" hx-target="#channels" hx-swap="outerHTML"> @@ -13,7 +13,7 @@ style="display: none;" hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}' hx-vals='{"net": "{{ item.net }}", "nick": "{{ item.nick }}"}' - hx-post="{% url 'nicks_insights' %}" + hx-post="{% url 'nicks_insights' index=index %}" hx-trigger="load" hx-target="#nicks" hx-swap="outerHTML"> @@ -81,7 +81,7 @@ {% if item.src == 'irc' %}