Pathogen
/
neptune
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Fix insights

This commit is contained in:
Mark Veidemanis 2023-01-14 16:36:00 +00:00
parent dbf581245b
commit 9ee9c7abde
Signed by: m
GPG Key ID: 5ACFCEED46C0904F
10 changed files with 126 additions and 111 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
app/urls.py
View File

@ -64,15 +64,14 @@ from core.views.ui.drilldown import ( # DrilldownTableView,; Drilldown,
DrilldownTableView, DrilldownTableView,
ThresholdInfoModal, ThresholdInfoModal,
) )
from core.views.ui.insights import (
# from core.views.ui.insights import ( Insights,
# Insights, InsightsChannels,
# InsightsChannels, InsightsInfoModal,
# InsightsInfoModal, InsightsMeta,
# InsightsMeta, InsightsNicks,
# InsightsNicks, InsightsSearch,
# InsightsSearch, )
# )
urlpatterns = [ urlpatterns = [
path("__debug__/", include("debug_toolbar.urls")), path("__debug__/", include("debug_toolbar.urls")),
@ -103,12 +102,32 @@ urlpatterns = [
path("context/", DrilldownContextModal.as_view(), name="modal_context"), path("context/", DrilldownContextModal.as_view(), name="modal_context"),
path("context_table/", DrilldownContextModal.as_view(), name="modal_context_table"), path("context_table/", DrilldownContextModal.as_view(), name="modal_context_table"),
## ##
# path("ui/insights/", Insights.as_view(), name="insights"), path("ui/insights/index/<str:index>/", Insights.as_view(), name="insights"),
# path("ui/insights/search/", InsightsSearch.as_view(), name="search_insights"), path(
# path("ui/insights/channels/", InsightsChannels.as_view(), name="chans_insights"), "ui/insights/index/<str:index>/search/",
# path("ui/insights/nicks/", InsightsNicks.as_view(), name="nicks_insights"), InsightsSearch.as_view(),
# path("ui/insights/meta/", InsightsMeta.as_view(), name="meta_insights"), name="search_insights",
# path("ui/insights/modal/", InsightsInfoModal.as_view(), name="modal_insights"), ),
path(
"ui/insights/index/<str:index>/channels/",
InsightsChannels.as_view(),
name="chans_insights",
),
path(
"ui/insights/index/<str:index>/nicks/",
InsightsNicks.as_view(),
name="nicks_insights",
),
path(
"ui/insights/index/<str:index>/meta/",
InsightsMeta.as_view(),
name="meta_insights",
),
path(
"ui/insights/index/<str:index>/modal/",
InsightsInfoModal.as_view(),
name="modal_insights",
),
## ##
path( path(
"manage/threshold/irc/overview/", "manage/threshold/irc/overview/",

25
core/lib/meta.py
View File

@ -3,7 +3,7 @@ from math import ceil
from django.conf import settings from django.conf import settings
from numpy import array_split from numpy import array_split
from core.db.elastic import client, run_main_query from core.db.storage import db
def construct_query(net, nicks): def construct_query(net, nicks):
@ -43,27 +43,14 @@ def get_meta(request, net, nicks, iter=True):
break break
meta_tmp = [] meta_tmp = []
query = construct_query(net, nicks_chunked) query = construct_query(net, nicks_chunked)
results = run_main_query( results = db.query(
client,
request.user, request.user,
query, query,
custom_query=True, index=settings.INDEX_META,
index=settings.ELASTICSEARCH_INDEX_META,
) )
if "hits" in results.keys(): if "object_list" in results.keys():
if "hits" in results["hits"]: for element in results["object_list"]:
for item in results["hits"]["hits"]: meta_tmp.append(element)
element = item["_source"]
element["id"] = item["_id"]
# Split the timestamp into date and time
ts = element["ts"]
ts_spl = ts.split("T")
date = ts_spl[0]
time = ts_spl[1]
element["date"] = date
element["time"] = time
meta_tmp.append(element)
for x in meta_tmp: for x in meta_tmp:
if x not in meta: if x not in meta:
meta.append(x) meta.append(x)

4
core/lib/nicktrace.py
View File

@ -3,7 +3,7 @@ from math import ceil
from django.conf import settings from django.conf import settings
from numpy import array_split from numpy import array_split
from core.lib.druid import client, run_main_query from core.db.storage import db
def construct_query(net, nicks): def construct_query(net, nicks):
@ -45,7 +45,7 @@ def get_nicks(request, net, nicks, iter=True):
if len(nicks_chunked) == 0: if len(nicks_chunked) == 0:
break break
query = construct_query(net, nicks_chunked) query = construct_query(net, nicks_chunked)
results = run_main_query(client, request.user, query, custom_query=True) results = db.query(request.user, query)
if "hits" in results.keys(): if "hits" in results.keys():
if "hits" in results["hits"]: if "hits" in results["hits"]:
for item in results["hits"]["hits"]: for item in results["hits"]["hits"]:

42
core/models.py
View File

@ -1,6 +1,7 @@
import logging import logging
import stripe import stripe
from django.conf import settings
from django.contrib.auth.models import AbstractUser from django.contrib.auth.models import AbstractUser
from django.db import models from django.db import models
from yaml import load from yaml import load
@ -24,15 +25,13 @@ PRIORITY_CHOICES = (
) )
INTERVAL_CHOICES = ( INTERVAL_CHOICES = (
("ondemand", "On demand"), (0, "On demand"),
("minute", "Every minute"), (60, "Every minute"),
("15m", "Every 15 minutes"), (900, "Every 15 minutes"),
("30m", "Every 30 minutes"), (1800, "Every 30 minutes"),
("hour", "Every hour"), (3600, "Every hour"),
("4h", "Every 4 hours"), (14400, "Every 4 hours"),
("day", "Every day"), (86400, "Every day"),
("week", "Every week"),
("month", "Every month"),
) )
@ -90,6 +89,19 @@ class User(AbstractUser):
def get_notification_settings(self): def get_notification_settings(self):
return NotificationSettings.objects.get_or_create(user=self)[0] return NotificationSettings.objects.get_or_create(user=self)[0]
@property
def allowed_indices(self):
indices = [settings.INDEX_MAIN]
if self.has_perm("core.index_meta"):
indices.append(settings.INDEX_META)
if self.has_perm("core.index_internal"):
indices.append(settings.INDEX_INT)
if self.has_perm("core.index_restricted"):
if self.has_perm("core.restricted_sources"):
indices.append(settings.INDEX_RESTRICTED)
return indices
class Session(models.Model): class Session(models.Model):
user = models.ForeignKey(User, on_delete=models.CASCADE) user = models.ForeignKey(User, on_delete=models.CASCADE)
@ -137,16 +149,10 @@ class ContentBlock(models.Model):
class Perms(models.Model): class Perms(models.Model):
class Meta: class Meta:
permissions = ( permissions = (
("bypass_hashing", "Can bypass field hashing"), #
("bypass_blacklist", "Can bypass the blacklist"), #
("bypass_encryption", "Can bypass field encryption"), #
("bypass_obfuscation", "Can bypass field obfuscation"), #
("bypass_delay", "Can bypass data delay"), #
("bypass_randomisation", "Can bypass data randomisation"), #
("post_irc", "Can post to IRC"), ("post_irc", "Can post to IRC"),
("post_discord", "Can post to Discord"), ("post_discord", "Can post to Discord"),
("query_search", "Can search with query strings"), #
("use_insights", "Can use the Insights page"), ("use_insights", "Can use the Insights page"),
("use_rules", "Can use the Rules page"),
("index_internal", "Can use the internal index"), ("index_internal", "Can use the internal index"),
("index_meta", "Can use the meta index"), ("index_meta", "Can use the meta index"),
("index_restricted", "Can use the restricted index"), ("index_restricted", "Can use the restricted index"),
@ -159,9 +165,7 @@ class NotificationRule(models.Model):
name = models.CharField(max_length=255) name = models.CharField(max_length=255)
priority = models.IntegerField(choices=PRIORITY_CHOICES, default=1) priority = models.IntegerField(choices=PRIORITY_CHOICES, default=1)
topic = models.CharField(max_length=255, null=True, blank=True) topic = models.CharField(max_length=255, null=True, blank=True)
interval = models.CharField( interval = models.IntegerField(choices=INTERVAL_CHOICES, default=0)
choices=INTERVAL_CHOICES, max_length=255, default="ondemand"
)
window = models.CharField(max_length=255, null=True, blank=True) window = models.CharField(max_length=255, null=True, blank=True)
enabled = models.BooleanField(default=True) enabled = models.BooleanField(default=True)
data = models.TextField() data = models.TextField()

18
core/templates/base.html
View File

@ -286,9 +286,21 @@
{% endif %} {% endif %}
{% if perms.core.use_insights %} {% if perms.core.use_insights %}
<a class="navbar-item" href="{# url 'insights' #}"> <div class="navbar-item has-dropdown is-hoverable">
Insights <a class="navbar-link">
</a> Insights
</a>
<div class="navbar-dropdown">
{% for index in user.allowed_indices %}
{% if index != "meta" %}
<a class="navbar-item" href="{% url 'insights' index=index %}">
{{ index }}
</a>
{% endif %}
{% endfor %}
</div>
</div>
{% endif %} {% endif %}
<a class="navbar-item add-button"> <a class="navbar-item add-button">
Install Install

4
core/templates/partials/rule-list.html
View File

@ -11,6 +11,8 @@
<th>id</th> <th>id</th>
<th>user</th> <th>user</th>
<th>name</th> <th>name</th>
<th>interval</th>
<th>window</th>
<th>priority</th> <th>priority</th>
<th>topic</th> <th>topic</th>
<th>enabled</th> <th>enabled</th>
@ -22,6 +24,8 @@
<td>{{ item.id }}</td> <td>{{ item.id }}</td>
<td>{{ item.user }}</td> <td>{{ item.user }}</td>
<td>{{ item.name }}</td> <td>{{ item.name }}</td>
<td>{{ item.interval }}s</td>
<td>{{ item.window|default_if_none:"—" }}</td>
<td>{{ item.priority }}</td> <td>{{ item.priority }}</td>
<td>{{ item.topic|default_if_none:"—" }}</td> <td>{{ item.topic|default_if_none:"—" }}</td>
<td> <td>

6
core/templates/ui/insights/info.html
View File

@ -4,7 +4,7 @@
style="display: none;" style="display: none;"
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}' hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
hx-vals='{"net": "{{ item.net }}", "nick": "{{ item.nick }}"}' hx-vals='{"net": "{{ item.net }}", "nick": "{{ item.nick }}"}'
hx-post="{% url 'chans_insights' %}" hx-post="{% url 'chans_insights' index=index %}"
hx-trigger="load" hx-trigger="load"
hx-target="#channels" hx-target="#channels"
hx-swap="outerHTML"> hx-swap="outerHTML">
@ -13,7 +13,7 @@
style="display: none;" style="display: none;"
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}' hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
hx-vals='{"net": "{{ item.net }}", "nick": "{{ item.nick }}"}' hx-vals='{"net": "{{ item.net }}", "nick": "{{ item.nick }}"}'
hx-post="{% url 'nicks_insights' %}" hx-post="{% url 'nicks_insights' index=index %}"
hx-trigger="load" hx-trigger="load"
hx-target="#nicks" hx-target="#nicks"
hx-swap="outerHTML"> hx-swap="outerHTML">
@ -81,7 +81,7 @@
{% if item.src == 'irc' %} {% if item.src == 'irc' %}
<button <button
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}' hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
hx-post="{% url 'modal_insights' %}" hx-post="{% url 'modal_insights' index=index %}"
hx-vals='{"net": "{{ item.net }}", "nick": "{{ item.nick }}", "channel": "{{ item.channel }}"}' hx-vals='{"net": "{{ item.net }}", "nick": "{{ item.nick }}", "channel": "{{ item.channel }}"}'
hx-target="#modals-here" hx-target="#modals-here"
hx-trigger="click" hx-trigger="click"

38
core/templates/ui/insights/insights.html
View File

@ -2,39 +2,7 @@
{% load static %} {% load static %}
{% block content %} {% block content %}
{% include 'partials/notify.html' %} {% include 'partials/notify.html' %}
<script> <script src="{% static 'tabs.js' %}"></script>
// tabbed browsing for the modal
function initTabs() {
TABS.forEach((tab) => {
tab.addEventListener('click', (e) => {
let selected = tab.getAttribute('data-tab');
updateActiveTab(tab);
updateActiveContent(selected);
})
})
}
function updateActiveTab(selected) {
TABS.forEach((tab) => {
if (tab && tab.classList.contains(ACTIVE_CLASS)) {
tab.classList.remove(ACTIVE_CLASS);
}
});
selected.classList.add(ACTIVE_CLASS);
}
function updateActiveContent(selected) {
CONTENT.forEach((item) => {
if (item && item.classList.contains(ACTIVE_CLASS)) {
item.classList.remove(ACTIVE_CLASS);
}
let data = item.getAttribute('data-content');
if (data === selected) {
item.classList.add(ACTIVE_CLASS);
}
});
}
</script>
<style> <style>
.icon { border-bottom: 0px !important;} .icon { border-bottom: 0px !important;}
</style> </style>
@ -47,7 +15,7 @@
{% csrf_token %} {% csrf_token %}
<div class="field has-addons"> <div class="field has-addons">
<div class="control is-expanded has-icons-left"> <div class="control is-expanded has-icons-left">
<input id="query_full" name="query_full" class="input" type="text" placeholder="nickname"> <input id="query_full" name="query" class="input" type="text" placeholder="nickname">
<span class="icon is-small is-left"> <span class="icon is-small is-left">
<i class="fas fa-magnifying-glass"></i> <i class="fas fa-magnifying-glass"></i>
</span> </span>
@ -55,7 +23,7 @@
<div class="control"> <div class="control">
<button <button
class="button is-info is-fullwidth" class="button is-info is-fullwidth"
hx-post="{% url 'search_insights' %}" hx-post="{% url 'search_insights' index=index %}"
hx-trigger="click" hx-trigger="click"
hx-target="#info" hx-target="#info"
hx-swap="outerHTML"> hx-swap="outerHTML">

2
core/templates/ui/insights/nicks.html
View File

@ -3,7 +3,7 @@
style="display: none;" style="display: none;"
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}' hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
hx-vals='{"net": "{{ net }}", "nicks": "{{ nicks }}"}' hx-vals='{"net": "{{ net }}", "nicks": "{{ nicks }}"}'
hx-post="{% url 'meta_insights' %}" hx-post="{% url 'meta_insights' index=index %}"
hx-trigger="load" hx-trigger="load"
hx-target="#meta" hx-target="#meta"
hx-swap="outerHTML"> hx-swap="outerHTML">

49
core/views/ui/insights.py
View File

@ -7,7 +7,7 @@ from django.views import View
from rest_framework.parsers import FormParser from rest_framework.parsers import FormParser
from rest_framework.views import APIView from rest_framework.views import APIView
from core.db.druid import query_single_result from core.db.storage import db
from core.lib.meta import get_meta from core.lib.meta import get_meta
from core.lib.nicktrace import get_nicks from core.lib.nicktrace import get_nicks
from core.lib.threshold import ( from core.lib.threshold import (
@ -23,8 +23,9 @@ class Insights(LoginRequiredMixin, PermissionRequiredMixin, View):
template_name = "ui/insights/insights.html" template_name = "ui/insights/insights.html"
permission_required = "use_insights" permission_required = "use_insights"
def get(self, request): def get(self, request, index):
return render(request, self.template_name) context = {"index": index}
return render(request, self.template_name, context)
class InsightsSearch(LoginRequiredMixin, PermissionRequiredMixin, View): class InsightsSearch(LoginRequiredMixin, PermissionRequiredMixin, View):
@ -32,13 +33,16 @@ class InsightsSearch(LoginRequiredMixin, PermissionRequiredMixin, View):
template_name = "ui/insights/info.html" template_name = "ui/insights/info.html"
permission_required = "use_insights" permission_required = "use_insights"
def post(self, request): def post(self, request, index):
query_params = request.POST.dict() query_params = request.POST.dict()
if "query_full" in query_params: if "query" in query_params:
query_params["query_full"] = "nick: " + query_params["query_full"] query_params["query"] = "nick: " + query_params["query"]
context = query_single_result(request, query_params) query_params["source"] = "all"
query_params["index"] = index
context = db.query_single_result(request, query_params)
if not context: if not context:
return HttpResponseForbidden() return HttpResponseForbidden()
context["index"] = index
return render(request, self.template_name, context) return render(request, self.template_name, context)
@ -47,7 +51,7 @@ class InsightsChannels(LoginRequiredMixin, PermissionRequiredMixin, APIView):
template_name = "ui/insights/channels.html" template_name = "ui/insights/channels.html"
permission_required = "use_insights" permission_required = "use_insights"
def post(self, request): def post(self, request, index):
if "net" not in request.data: if "net" not in request.data:
return HttpResponse("No net") return HttpResponse("No net")
if "nick" not in request.data: if "nick" not in request.data:
@ -58,7 +62,13 @@ class InsightsChannels(LoginRequiredMixin, PermissionRequiredMixin, APIView):
num_users = annotate_num_users(net, chans) num_users = annotate_num_users(net, chans)
if not chans: if not chans:
return HttpResponseForbidden() return HttpResponseForbidden()
context = {"net": net, "nick": nick, "chans": chans, "num_users": num_users} context = {
"net": net,
"nick": nick,
"chans": chans,
"num_users": num_users,
"index": index,
}
return render(request, self.template_name, context) return render(request, self.template_name, context)
@ -67,7 +77,7 @@ class InsightsNicks(LoginRequiredMixin, PermissionRequiredMixin, APIView):
template_name = "ui/insights/nicks.html" template_name = "ui/insights/nicks.html"
permission_required = "use_insights" permission_required = "use_insights"
def post(self, request): def post(self, request, index):
if "net" not in request.data: if "net" not in request.data:
return HttpResponse("No net") return HttpResponse("No net")
if "nick" not in request.data: if "nick" not in request.data:
@ -82,7 +92,13 @@ class InsightsNicks(LoginRequiredMixin, PermissionRequiredMixin, APIView):
online = annotate_online(net, nicks) online = annotate_online(net, nicks)
if not nicks: if not nicks:
return HttpResponseForbidden() return HttpResponseForbidden()
context = {"net": net, "nick": nick, "nicks": nicks, "online": online} context = {
"net": net,
"nick": nick,
"nicks": nicks,
"online": online,
"index": index,
}
return render(request, self.template_name, context) return render(request, self.template_name, context)
@ -91,7 +107,7 @@ class InsightsMeta(LoginRequiredMixin, PermissionRequiredMixin, APIView):
template_name = "ui/insights/meta.html" template_name = "ui/insights/meta.html"
permission_required = "use_insights" permission_required = "use_insights"
def post(self, request): def post(self, request, index):
if "net" not in request.data: if "net" not in request.data:
return HttpResponse("No net") return HttpResponse("No net")
if "nicks" not in request.data: if "nicks" not in request.data:
@ -99,6 +115,10 @@ class InsightsMeta(LoginRequiredMixin, PermissionRequiredMixin, APIView):
net = request.data["net"] net = request.data["net"]
nicks = request.data["nicks"] nicks = request.data["nicks"]
nicks = literal_eval(nicks) nicks = literal_eval(nicks)
# Check the user has permissions to use the meta index
if not request.user.has_perm("core.index_meta"):
return HttpResponseForbidden()
meta = get_meta(request, net, nicks) meta = get_meta(request, net, nicks)
unique_values = {} unique_values = {}
# Create a map of unique values for each key for each nick # Create a map of unique values for each key for each nick
@ -122,7 +142,7 @@ class InsightsMeta(LoginRequiredMixin, PermissionRequiredMixin, APIView):
meta_dedup[k].add(v) meta_dedup[k].add(v)
unique_values[nick][k].remove(v) unique_values[nick][k].remove(v)
context = {"net": net, "nicks": nicks, "meta": meta_dedup} context = {"net": net, "nicks": nicks, "meta": meta_dedup, "index": index}
return render(request, self.template_name, context) return render(request, self.template_name, context)
@ -131,7 +151,7 @@ class InsightsInfoModal(LoginRequiredMixin, PermissionRequiredMixin, APIView):
template_name = "modals/drilldown.html" template_name = "modals/drilldown.html"
permission_required = "use_insights" permission_required = "use_insights"
def post(self, request): def post(self, request, index):
if "net" not in request.data: if "net" not in request.data:
return JsonResponse({"success": False}) return JsonResponse({"success": False})
if "nick" not in request.data: if "nick" not in request.data:
@ -163,5 +183,6 @@ class InsightsInfoModal(LoginRequiredMixin, PermissionRequiredMixin, APIView):
"inter_users": inter_users, "inter_users": inter_users,
"num_users": num_users, "num_users": num_users,
"num_chans": num_chans, "num_chans": num_chans,
"index": index,
} }
return render(request, self.template_name, context) return render(request, self.template_name, context)