Pathogen
/
neptune
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Fix insights

This commit is contained in:
Mark Veidemanis 2023-01-14 16:36:00 +00:00
parent dbf581245b
commit 9ee9c7abde
Signed by: m
GPG Key ID: 5ACFCEED46C0904F
10 changed files with 126 additions and 111 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
app/urls.py
View File

@ -64,15 +64,14 @@ from core.views.ui.drilldown import ( # DrilldownTableView,; Drilldown,
DrilldownTableView,
ThresholdInfoModal,
)
# from core.views.ui.insights import (
# Insights,
# InsightsChannels,
# InsightsInfoModal,
# InsightsMeta,
# InsightsNicks,
# InsightsSearch,
# )
from core.views.ui.insights import (
Insights,
InsightsChannels,
InsightsInfoModal,
InsightsMeta,
InsightsNicks,
InsightsSearch,
)
urlpatterns = [
path("__debug__/", include("debug_toolbar.urls")),
@ -103,12 +102,32 @@ urlpatterns = [
path("context/", DrilldownContextModal.as_view(), name="modal_context"),
path("context_table/", DrilldownContextModal.as_view(), name="modal_context_table"),
##
# path("ui/insights/", Insights.as_view(), name="insights"),
# path("ui/insights/search/", InsightsSearch.as_view(), name="search_insights"),
# path("ui/insights/channels/", InsightsChannels.as_view(), name="chans_insights"),
# path("ui/insights/nicks/", InsightsNicks.as_view(), name="nicks_insights"),
# path("ui/insights/meta/", InsightsMeta.as_view(), name="meta_insights"),
# path("ui/insights/modal/", InsightsInfoModal.as_view(), name="modal_insights"),
path("ui/insights/index/<str:index>/", Insights.as_view(), name="insights"),
path(
"ui/insights/index/<str:index>/search/",
InsightsSearch.as_view(),
name="search_insights",
),
path(
"ui/insights/index/<str:index>/channels/",
InsightsChannels.as_view(),
name="chans_insights",
),
path(
"ui/insights/index/<str:index>/nicks/",
InsightsNicks.as_view(),
name="nicks_insights",
),
path(
"ui/insights/index/<str:index>/meta/",
InsightsMeta.as_view(),
name="meta_insights",
),
path(
"ui/insights/index/<str:index>/modal/",
InsightsInfoModal.as_view(),
name="modal_insights",
),
##
path(
"manage/threshold/irc/overview/",

25
core/lib/meta.py
View File

@ -3,7 +3,7 @@ from math import ceil
from django.conf import settings
from numpy import array_split
from core.db.elastic import client, run_main_query
from core.db.storage import db
def construct_query(net, nicks):
@ -43,27 +43,14 @@ def get_meta(request, net, nicks, iter=True):
break
meta_tmp = []
query = construct_query(net, nicks_chunked)
results = run_main_query(
client,
results = db.query(
request.user,
query,
custom_query=True,
index=settings.ELASTICSEARCH_INDEX_META,
index=settings.INDEX_META,
)
if "hits" in results.keys():
if "hits" in results["hits"]:
for item in results["hits"]["hits"]:
element = item["_source"]
element["id"] = item["_id"]
# Split the timestamp into date and time
ts = element["ts"]
ts_spl = ts.split("T")
date = ts_spl[0]
time = ts_spl[1]
element["date"] = date
element["time"] = time
meta_tmp.append(element)
if "object_list" in results.keys():
for element in results["object_list"]:
meta_tmp.append(element)
for x in meta_tmp:
if x not in meta:
meta.append(x)

4
core/lib/nicktrace.py
View File

@ -3,7 +3,7 @@ from math import ceil
from django.conf import settings
from numpy import array_split
from core.lib.druid import client, run_main_query
from core.db.storage import db
def construct_query(net, nicks):
@ -45,7 +45,7 @@ def get_nicks(request, net, nicks, iter=True):
if len(nicks_chunked) == 0:
break
query = construct_query(net, nicks_chunked)
results = run_main_query(client, request.user, query, custom_query=True)
results = db.query(request.user, query)
if "hits" in results.keys():
if "hits" in results["hits"]:
for item in results["hits"]["hits"]:

42
core/models.py
View File

@ -1,6 +1,7 @@
import logging
import stripe
from django.conf import settings
from django.contrib.auth.models import AbstractUser
from django.db import models
from yaml import load
@ -24,15 +25,13 @@ PRIORITY_CHOICES = (
)
INTERVAL_CHOICES = (
("ondemand", "On demand"),
("minute", "Every minute"),
("15m", "Every 15 minutes"),
("30m", "Every 30 minutes"),
("hour", "Every hour"),
("4h", "Every 4 hours"),
("day", "Every day"),
("week", "Every week"),
("month", "Every month"),
(0, "On demand"),
(60, "Every minute"),
(900, "Every 15 minutes"),
(1800, "Every 30 minutes"),
(3600, "Every hour"),
(14400, "Every 4 hours"),
(86400, "Every day"),
)
@ -90,6 +89,19 @@ class User(AbstractUser):
def get_notification_settings(self):
return NotificationSettings.objects.get_or_create(user=self)[0]
@property
def allowed_indices(self):
indices = [settings.INDEX_MAIN]
if self.has_perm("core.index_meta"):
indices.append(settings.INDEX_META)
if self.has_perm("core.index_internal"):
indices.append(settings.INDEX_INT)
if self.has_perm("core.index_restricted"):
if self.has_perm("core.restricted_sources"):
indices.append(settings.INDEX_RESTRICTED)
return indices
class Session(models.Model):
user = models.ForeignKey(User, on_delete=models.CASCADE)
@ -137,16 +149,10 @@ class ContentBlock(models.Model):
class Perms(models.Model):
class Meta:
permissions = (
("bypass_hashing", "Can bypass field hashing"), #
("bypass_blacklist", "Can bypass the blacklist"), #
("bypass_encryption", "Can bypass field encryption"), #
("bypass_obfuscation", "Can bypass field obfuscation"), #
("bypass_delay", "Can bypass data delay"), #
("bypass_randomisation", "Can bypass data randomisation"), #
("post_irc", "Can post to IRC"),
("post_discord", "Can post to Discord"),
("query_search", "Can search with query strings"), #
("use_insights", "Can use the Insights page"),
("use_rules", "Can use the Rules page"),
("index_internal", "Can use the internal index"),
("index_meta", "Can use the meta index"),
("index_restricted", "Can use the restricted index"),
@ -159,9 +165,7 @@ class NotificationRule(models.Model):
name = models.CharField(max_length=255)
priority = models.IntegerField(choices=PRIORITY_CHOICES, default=1)
topic = models.CharField(max_length=255, null=True, blank=True)
interval = models.CharField(
choices=INTERVAL_CHOICES, max_length=255, default="ondemand"
)
interval = models.IntegerField(choices=INTERVAL_CHOICES, default=0)
window = models.CharField(max_length=255, null=True, blank=True)
enabled = models.BooleanField(default=True)
data = models.TextField()

18
core/templates/base.html
View File

@ -286,9 +286,21 @@
{% endif %}
{% if perms.core.use_insights %}
<a class="navbar-item" href="{# url 'insights' #}">
Insights
</a>
<div class="navbar-item has-dropdown is-hoverable">
<a class="navbar-link">
Insights
</a>
<div class="navbar-dropdown">
{% for index in user.allowed_indices %}
{% if index != "meta" %}
<a class="navbar-item" href="{% url 'insights' index=index %}">
{{ index }}
</a>
{% endif %}
{% endfor %}
</div>
</div>
{% endif %}
<a class="navbar-item add-button">
Install

4
core/templates/partials/rule-list.html
View File

@ -11,6 +11,8 @@
<th>id</th>
<th>user</th>
<th>name</th>
<th>interval</th>
<th>window</th>
<th>priority</th>
<th>topic</th>
<th>enabled</th>
@ -22,6 +24,8 @@
<td>{{ item.id }}</td>
<td>{{ item.user }}</td>
<td>{{ item.name }}</td>
<td>{{ item.interval }}s</td>
<td>{{ item.window|default_if_none:"—" }}</td>
<td>{{ item.priority }}</td>
<td>{{ item.topic|default_if_none:"—" }}</td>
<td>

6
core/templates/ui/insights/info.html
View File

@ -4,7 +4,7 @@
style="display: none;"
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
hx-vals='{"net": "{{ item.net }}", "nick": "{{ item.nick }}"}'
hx-post="{% url 'chans_insights' %}"
hx-post="{% url 'chans_insights' index=index %}"
hx-trigger="load"
hx-target="#channels"
hx-swap="outerHTML">
@ -13,7 +13,7 @@
style="display: none;"
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
hx-vals='{"net": "{{ item.net }}", "nick": "{{ item.nick }}"}'
hx-post="{% url 'nicks_insights' %}"
hx-post="{% url 'nicks_insights' index=index %}"
hx-trigger="load"
hx-target="#nicks"
hx-swap="outerHTML">
@ -81,7 +81,7 @@
{% if item.src == 'irc' %}
<button
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
hx-post="{% url 'modal_insights' %}"
hx-post="{% url 'modal_insights' index=index %}"
hx-vals='{"net": "{{ item.net }}", "nick": "{{ item.nick }}", "channel": "{{ item.channel }}"}'
hx-target="#modals-here"
hx-trigger="click"

38
core/templates/ui/insights/insights.html
View File

@ -2,39 +2,7 @@
{% load static %}
{% block content %}
{% include 'partials/notify.html' %}
<script>
// tabbed browsing for the modal
function initTabs() {
TABS.forEach((tab) => {
tab.addEventListener('click', (e) => {
let selected = tab.getAttribute('data-tab');
updateActiveTab(tab);
updateActiveContent(selected);
})
})
}
function updateActiveTab(selected) {
TABS.forEach((tab) => {
if (tab && tab.classList.contains(ACTIVE_CLASS)) {
tab.classList.remove(ACTIVE_CLASS);
}
});
selected.classList.add(ACTIVE_CLASS);
}
function updateActiveContent(selected) {
CONTENT.forEach((item) => {
if (item && item.classList.contains(ACTIVE_CLASS)) {
item.classList.remove(ACTIVE_CLASS);
}
let data = item.getAttribute('data-content');
if (data === selected) {
item.classList.add(ACTIVE_CLASS);
}
});
}
</script>
<script src="{% static 'tabs.js' %}"></script>
<style>
.icon { border-bottom: 0px !important;}
</style>
@ -47,7 +15,7 @@
{% csrf_token %}
<div class="field has-addons">
<div class="control is-expanded has-icons-left">
<input id="query_full" name="query_full" class="input" type="text" placeholder="nickname">
<input id="query_full" name="query" class="input" type="text" placeholder="nickname">
<span class="icon is-small is-left">
<i class="fas fa-magnifying-glass"></i>
</span>
@ -55,7 +23,7 @@
<div class="control">
<button
class="button is-info is-fullwidth"
hx-post="{% url 'search_insights' %}"
hx-post="{% url 'search_insights' index=index %}"
hx-trigger="click"
hx-target="#info"
hx-swap="outerHTML">

2
core/templates/ui/insights/nicks.html
View File

@ -3,7 +3,7 @@
style="display: none;"
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
hx-vals='{"net": "{{ net }}", "nicks": "{{ nicks }}"}'
hx-post="{% url 'meta_insights' %}"
hx-post="{% url 'meta_insights' index=index %}"
hx-trigger="load"
hx-target="#meta"
hx-swap="outerHTML">

49
core/views/ui/insights.py
View File

@ -7,7 +7,7 @@ from django.views import View
from rest_framework.parsers import FormParser
from rest_framework.views import APIView
from core.db.druid import query_single_result
from core.db.storage import db
from core.lib.meta import get_meta
from core.lib.nicktrace import get_nicks
from core.lib.threshold import (
@ -23,8 +23,9 @@ class Insights(LoginRequiredMixin, PermissionRequiredMixin, View):
template_name = "ui/insights/insights.html"
permission_required = "use_insights"
def get(self, request):
return render(request, self.template_name)
def get(self, request, index):
context = {"index": index}
return render(request, self.template_name, context)
class InsightsSearch(LoginRequiredMixin, PermissionRequiredMixin, View):
@ -32,13 +33,16 @@ class InsightsSearch(LoginRequiredMixin, PermissionRequiredMixin, View):
template_name = "ui/insights/info.html"
permission_required = "use_insights"
def post(self, request):
def post(self, request, index):
query_params = request.POST.dict()
if "query_full" in query_params:
query_params["query_full"] = "nick: " + query_params["query_full"]
context = query_single_result(request, query_params)
if "query" in query_params:
query_params["query"] = "nick: " + query_params["query"]
query_params["source"] = "all"
query_params["index"] = index
context = db.query_single_result(request, query_params)
if not context:
return HttpResponseForbidden()
context["index"] = index
return render(request, self.template_name, context)
@ -47,7 +51,7 @@ class InsightsChannels(LoginRequiredMixin, PermissionRequiredMixin, APIView):
template_name = "ui/insights/channels.html"
permission_required = "use_insights"
def post(self, request):
def post(self, request, index):
if "net" not in request.data:
return HttpResponse("No net")
if "nick" not in request.data:
@ -58,7 +62,13 @@ class InsightsChannels(LoginRequiredMixin, PermissionRequiredMixin, APIView):
num_users = annotate_num_users(net, chans)
if not chans:
return HttpResponseForbidden()
context = {"net": net, "nick": nick, "chans": chans, "num_users": num_users}
context = {
"net": net,
"nick": nick,
"chans": chans,
"num_users": num_users,
"index": index,
}
return render(request, self.template_name, context)
@ -67,7 +77,7 @@ class InsightsNicks(LoginRequiredMixin, PermissionRequiredMixin, APIView):
template_name = "ui/insights/nicks.html"
permission_required = "use_insights"
def post(self, request):
def post(self, request, index):
if "net" not in request.data:
return HttpResponse("No net")
if "nick" not in request.data:
@ -82,7 +92,13 @@ class InsightsNicks(LoginRequiredMixin, PermissionRequiredMixin, APIView):
online = annotate_online(net, nicks)
if not nicks:
return HttpResponseForbidden()
context = {"net": net, "nick": nick, "nicks": nicks, "online": online}
context = {
"net": net,
"nick": nick,
"nicks": nicks,
"online": online,
"index": index,
}
return render(request, self.template_name, context)
@ -91,7 +107,7 @@ class InsightsMeta(LoginRequiredMixin, PermissionRequiredMixin, APIView):
template_name = "ui/insights/meta.html"
permission_required = "use_insights"
def post(self, request):
def post(self, request, index):
if "net" not in request.data:
return HttpResponse("No net")
if "nicks" not in request.data:
@ -99,6 +115,10 @@ class InsightsMeta(LoginRequiredMixin, PermissionRequiredMixin, APIView):
net = request.data["net"]
nicks = request.data["nicks"]
nicks = literal_eval(nicks)
# Check the user has permissions to use the meta index
if not request.user.has_perm("core.index_meta"):
return HttpResponseForbidden()
meta = get_meta(request, net, nicks)
unique_values = {}
# Create a map of unique values for each key for each nick
@ -122,7 +142,7 @@ class InsightsMeta(LoginRequiredMixin, PermissionRequiredMixin, APIView):
meta_dedup[k].add(v)
unique_values[nick][k].remove(v)
context = {"net": net, "nicks": nicks, "meta": meta_dedup}
context = {"net": net, "nicks": nicks, "meta": meta_dedup, "index": index}
return render(request, self.template_name, context)
@ -131,7 +151,7 @@ class InsightsInfoModal(LoginRequiredMixin, PermissionRequiredMixin, APIView):
template_name = "modals/drilldown.html"
permission_required = "use_insights"
def post(self, request):
def post(self, request, index):
if "net" not in request.data:
return JsonResponse({"success": False})
if "nick" not in request.data:
@ -163,5 +183,6 @@ class InsightsInfoModal(LoginRequiredMixin, PermissionRequiredMixin, APIView):
"inter_users": inter_users,
"num_users": num_users,
"num_chans": num_chans,
"index": index,
}
return render(request, self.template_name, context)