Fix insights
This commit is contained in:
parent
dbf581245b
commit
9ee9c7abde
49
app/urls.py
49
app/urls.py
|
@ -64,15 +64,14 @@ from core.views.ui.drilldown import ( # DrilldownTableView,; Drilldown,
|
|||
DrilldownTableView,
|
||||
ThresholdInfoModal,
|
||||
)
|
||||
|
||||
# from core.views.ui.insights import (
|
||||
# Insights,
|
||||
# InsightsChannels,
|
||||
# InsightsInfoModal,
|
||||
# InsightsMeta,
|
||||
# InsightsNicks,
|
||||
# InsightsSearch,
|
||||
# )
|
||||
from core.views.ui.insights import (
|
||||
Insights,
|
||||
InsightsChannels,
|
||||
InsightsInfoModal,
|
||||
InsightsMeta,
|
||||
InsightsNicks,
|
||||
InsightsSearch,
|
||||
)
|
||||
|
||||
urlpatterns = [
|
||||
path("__debug__/", include("debug_toolbar.urls")),
|
||||
|
@ -103,12 +102,32 @@ urlpatterns = [
|
|||
path("context/", DrilldownContextModal.as_view(), name="modal_context"),
|
||||
path("context_table/", DrilldownContextModal.as_view(), name="modal_context_table"),
|
||||
##
|
||||
# path("ui/insights/", Insights.as_view(), name="insights"),
|
||||
# path("ui/insights/search/", InsightsSearch.as_view(), name="search_insights"),
|
||||
# path("ui/insights/channels/", InsightsChannels.as_view(), name="chans_insights"),
|
||||
# path("ui/insights/nicks/", InsightsNicks.as_view(), name="nicks_insights"),
|
||||
# path("ui/insights/meta/", InsightsMeta.as_view(), name="meta_insights"),
|
||||
# path("ui/insights/modal/", InsightsInfoModal.as_view(), name="modal_insights"),
|
||||
path("ui/insights/index/<str:index>/", Insights.as_view(), name="insights"),
|
||||
path(
|
||||
"ui/insights/index/<str:index>/search/",
|
||||
InsightsSearch.as_view(),
|
||||
name="search_insights",
|
||||
),
|
||||
path(
|
||||
"ui/insights/index/<str:index>/channels/",
|
||||
InsightsChannels.as_view(),
|
||||
name="chans_insights",
|
||||
),
|
||||
path(
|
||||
"ui/insights/index/<str:index>/nicks/",
|
||||
InsightsNicks.as_view(),
|
||||
name="nicks_insights",
|
||||
),
|
||||
path(
|
||||
"ui/insights/index/<str:index>/meta/",
|
||||
InsightsMeta.as_view(),
|
||||
name="meta_insights",
|
||||
),
|
||||
path(
|
||||
"ui/insights/index/<str:index>/modal/",
|
||||
InsightsInfoModal.as_view(),
|
||||
name="modal_insights",
|
||||
),
|
||||
##
|
||||
path(
|
||||
"manage/threshold/irc/overview/",
|
||||
|
|
|
@ -3,7 +3,7 @@ from math import ceil
|
|||
from django.conf import settings
|
||||
from numpy import array_split
|
||||
|
||||
from core.db.elastic import client, run_main_query
|
||||
from core.db.storage import db
|
||||
|
||||
|
||||
def construct_query(net, nicks):
|
||||
|
@ -43,27 +43,14 @@ def get_meta(request, net, nicks, iter=True):
|
|||
break
|
||||
meta_tmp = []
|
||||
query = construct_query(net, nicks_chunked)
|
||||
results = run_main_query(
|
||||
client,
|
||||
results = db.query(
|
||||
request.user,
|
||||
query,
|
||||
custom_query=True,
|
||||
index=settings.ELASTICSEARCH_INDEX_META,
|
||||
index=settings.INDEX_META,
|
||||
)
|
||||
if "hits" in results.keys():
|
||||
if "hits" in results["hits"]:
|
||||
for item in results["hits"]["hits"]:
|
||||
element = item["_source"]
|
||||
element["id"] = item["_id"]
|
||||
|
||||
# Split the timestamp into date and time
|
||||
ts = element["ts"]
|
||||
ts_spl = ts.split("T")
|
||||
date = ts_spl[0]
|
||||
time = ts_spl[1]
|
||||
element["date"] = date
|
||||
element["time"] = time
|
||||
meta_tmp.append(element)
|
||||
if "object_list" in results.keys():
|
||||
for element in results["object_list"]:
|
||||
meta_tmp.append(element)
|
||||
for x in meta_tmp:
|
||||
if x not in meta:
|
||||
meta.append(x)
|
||||
|
|
|
@ -3,7 +3,7 @@ from math import ceil
|
|||
from django.conf import settings
|
||||
from numpy import array_split
|
||||
|
||||
from core.lib.druid import client, run_main_query
|
||||
from core.db.storage import db
|
||||
|
||||
|
||||
def construct_query(net, nicks):
|
||||
|
@ -45,7 +45,7 @@ def get_nicks(request, net, nicks, iter=True):
|
|||
if len(nicks_chunked) == 0:
|
||||
break
|
||||
query = construct_query(net, nicks_chunked)
|
||||
results = run_main_query(client, request.user, query, custom_query=True)
|
||||
results = db.query(request.user, query)
|
||||
if "hits" in results.keys():
|
||||
if "hits" in results["hits"]:
|
||||
for item in results["hits"]["hits"]:
|
||||
|
|
|
@ -1,6 +1,7 @@
|
|||
import logging
|
||||
|
||||
import stripe
|
||||
from django.conf import settings
|
||||
from django.contrib.auth.models import AbstractUser
|
||||
from django.db import models
|
||||
from yaml import load
|
||||
|
@ -24,15 +25,13 @@ PRIORITY_CHOICES = (
|
|||
)
|
||||
|
||||
INTERVAL_CHOICES = (
|
||||
("ondemand", "On demand"),
|
||||
("minute", "Every minute"),
|
||||
("15m", "Every 15 minutes"),
|
||||
("30m", "Every 30 minutes"),
|
||||
("hour", "Every hour"),
|
||||
("4h", "Every 4 hours"),
|
||||
("day", "Every day"),
|
||||
("week", "Every week"),
|
||||
("month", "Every month"),
|
||||
(0, "On demand"),
|
||||
(60, "Every minute"),
|
||||
(900, "Every 15 minutes"),
|
||||
(1800, "Every 30 minutes"),
|
||||
(3600, "Every hour"),
|
||||
(14400, "Every 4 hours"),
|
||||
(86400, "Every day"),
|
||||
)
|
||||
|
||||
|
||||
|
@ -90,6 +89,19 @@ class User(AbstractUser):
|
|||
def get_notification_settings(self):
|
||||
return NotificationSettings.objects.get_or_create(user=self)[0]
|
||||
|
||||
@property
|
||||
def allowed_indices(self):
|
||||
indices = [settings.INDEX_MAIN]
|
||||
if self.has_perm("core.index_meta"):
|
||||
indices.append(settings.INDEX_META)
|
||||
if self.has_perm("core.index_internal"):
|
||||
indices.append(settings.INDEX_INT)
|
||||
if self.has_perm("core.index_restricted"):
|
||||
if self.has_perm("core.restricted_sources"):
|
||||
indices.append(settings.INDEX_RESTRICTED)
|
||||
|
||||
return indices
|
||||
|
||||
|
||||
class Session(models.Model):
|
||||
user = models.ForeignKey(User, on_delete=models.CASCADE)
|
||||
|
@ -137,16 +149,10 @@ class ContentBlock(models.Model):
|
|||
class Perms(models.Model):
|
||||
class Meta:
|
||||
permissions = (
|
||||
("bypass_hashing", "Can bypass field hashing"), #
|
||||
("bypass_blacklist", "Can bypass the blacklist"), #
|
||||
("bypass_encryption", "Can bypass field encryption"), #
|
||||
("bypass_obfuscation", "Can bypass field obfuscation"), #
|
||||
("bypass_delay", "Can bypass data delay"), #
|
||||
("bypass_randomisation", "Can bypass data randomisation"), #
|
||||
("post_irc", "Can post to IRC"),
|
||||
("post_discord", "Can post to Discord"),
|
||||
("query_search", "Can search with query strings"), #
|
||||
("use_insights", "Can use the Insights page"),
|
||||
("use_rules", "Can use the Rules page"),
|
||||
("index_internal", "Can use the internal index"),
|
||||
("index_meta", "Can use the meta index"),
|
||||
("index_restricted", "Can use the restricted index"),
|
||||
|
@ -159,9 +165,7 @@ class NotificationRule(models.Model):
|
|||
name = models.CharField(max_length=255)
|
||||
priority = models.IntegerField(choices=PRIORITY_CHOICES, default=1)
|
||||
topic = models.CharField(max_length=255, null=True, blank=True)
|
||||
interval = models.CharField(
|
||||
choices=INTERVAL_CHOICES, max_length=255, default="ondemand"
|
||||
)
|
||||
interval = models.IntegerField(choices=INTERVAL_CHOICES, default=0)
|
||||
window = models.CharField(max_length=255, null=True, blank=True)
|
||||
enabled = models.BooleanField(default=True)
|
||||
data = models.TextField()
|
||||
|
|
|
@ -286,9 +286,21 @@
|
|||
{% endif %}
|
||||
|
||||
{% if perms.core.use_insights %}
|
||||
<a class="navbar-item" href="{# url 'insights' #}">
|
||||
Insights
|
||||
</a>
|
||||
<div class="navbar-item has-dropdown is-hoverable">
|
||||
<a class="navbar-link">
|
||||
Insights
|
||||
</a>
|
||||
|
||||
<div class="navbar-dropdown">
|
||||
{% for index in user.allowed_indices %}
|
||||
{% if index != "meta" %}
|
||||
<a class="navbar-item" href="{% url 'insights' index=index %}">
|
||||
{{ index }}
|
||||
</a>
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
<a class="navbar-item add-button">
|
||||
Install
|
||||
|
|
|
@ -11,6 +11,8 @@
|
|||
<th>id</th>
|
||||
<th>user</th>
|
||||
<th>name</th>
|
||||
<th>interval</th>
|
||||
<th>window</th>
|
||||
<th>priority</th>
|
||||
<th>topic</th>
|
||||
<th>enabled</th>
|
||||
|
@ -22,6 +24,8 @@
|
|||
<td>{{ item.id }}</td>
|
||||
<td>{{ item.user }}</td>
|
||||
<td>{{ item.name }}</td>
|
||||
<td>{{ item.interval }}s</td>
|
||||
<td>{{ item.window|default_if_none:"—" }}</td>
|
||||
<td>{{ item.priority }}</td>
|
||||
<td>{{ item.topic|default_if_none:"—" }}</td>
|
||||
<td>
|
||||
|
|
|
@ -4,7 +4,7 @@
|
|||
style="display: none;"
|
||||
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
|
||||
hx-vals='{"net": "{{ item.net }}", "nick": "{{ item.nick }}"}'
|
||||
hx-post="{% url 'chans_insights' %}"
|
||||
hx-post="{% url 'chans_insights' index=index %}"
|
||||
hx-trigger="load"
|
||||
hx-target="#channels"
|
||||
hx-swap="outerHTML">
|
||||
|
@ -13,7 +13,7 @@
|
|||
style="display: none;"
|
||||
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
|
||||
hx-vals='{"net": "{{ item.net }}", "nick": "{{ item.nick }}"}'
|
||||
hx-post="{% url 'nicks_insights' %}"
|
||||
hx-post="{% url 'nicks_insights' index=index %}"
|
||||
hx-trigger="load"
|
||||
hx-target="#nicks"
|
||||
hx-swap="outerHTML">
|
||||
|
@ -81,7 +81,7 @@
|
|||
{% if item.src == 'irc' %}
|
||||
<button
|
||||
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
|
||||
hx-post="{% url 'modal_insights' %}"
|
||||
hx-post="{% url 'modal_insights' index=index %}"
|
||||
hx-vals='{"net": "{{ item.net }}", "nick": "{{ item.nick }}", "channel": "{{ item.channel }}"}'
|
||||
hx-target="#modals-here"
|
||||
hx-trigger="click"
|
||||
|
|
|
@ -2,39 +2,7 @@
|
|||
{% load static %}
|
||||
{% block content %}
|
||||
{% include 'partials/notify.html' %}
|
||||
<script>
|
||||
// tabbed browsing for the modal
|
||||
function initTabs() {
|
||||
TABS.forEach((tab) => {
|
||||
tab.addEventListener('click', (e) => {
|
||||
let selected = tab.getAttribute('data-tab');
|
||||
updateActiveTab(tab);
|
||||
updateActiveContent(selected);
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
function updateActiveTab(selected) {
|
||||
TABS.forEach((tab) => {
|
||||
if (tab && tab.classList.contains(ACTIVE_CLASS)) {
|
||||
tab.classList.remove(ACTIVE_CLASS);
|
||||
}
|
||||
});
|
||||
selected.classList.add(ACTIVE_CLASS);
|
||||
}
|
||||
|
||||
function updateActiveContent(selected) {
|
||||
CONTENT.forEach((item) => {
|
||||
if (item && item.classList.contains(ACTIVE_CLASS)) {
|
||||
item.classList.remove(ACTIVE_CLASS);
|
||||
}
|
||||
let data = item.getAttribute('data-content');
|
||||
if (data === selected) {
|
||||
item.classList.add(ACTIVE_CLASS);
|
||||
}
|
||||
});
|
||||
}
|
||||
</script>
|
||||
<script src="{% static 'tabs.js' %}"></script>
|
||||
<style>
|
||||
.icon { border-bottom: 0px !important;}
|
||||
</style>
|
||||
|
@ -47,7 +15,7 @@
|
|||
{% csrf_token %}
|
||||
<div class="field has-addons">
|
||||
<div class="control is-expanded has-icons-left">
|
||||
<input id="query_full" name="query_full" class="input" type="text" placeholder="nickname">
|
||||
<input id="query_full" name="query" class="input" type="text" placeholder="nickname">
|
||||
<span class="icon is-small is-left">
|
||||
<i class="fas fa-magnifying-glass"></i>
|
||||
</span>
|
||||
|
@ -55,7 +23,7 @@
|
|||
<div class="control">
|
||||
<button
|
||||
class="button is-info is-fullwidth"
|
||||
hx-post="{% url 'search_insights' %}"
|
||||
hx-post="{% url 'search_insights' index=index %}"
|
||||
hx-trigger="click"
|
||||
hx-target="#info"
|
||||
hx-swap="outerHTML">
|
||||
|
|
|
@ -3,7 +3,7 @@
|
|||
style="display: none;"
|
||||
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
|
||||
hx-vals='{"net": "{{ net }}", "nicks": "{{ nicks }}"}'
|
||||
hx-post="{% url 'meta_insights' %}"
|
||||
hx-post="{% url 'meta_insights' index=index %}"
|
||||
hx-trigger="load"
|
||||
hx-target="#meta"
|
||||
hx-swap="outerHTML">
|
||||
|
|
|
@ -7,7 +7,7 @@ from django.views import View
|
|||
from rest_framework.parsers import FormParser
|
||||
from rest_framework.views import APIView
|
||||
|
||||
from core.db.druid import query_single_result
|
||||
from core.db.storage import db
|
||||
from core.lib.meta import get_meta
|
||||
from core.lib.nicktrace import get_nicks
|
||||
from core.lib.threshold import (
|
||||
|
@ -23,8 +23,9 @@ class Insights(LoginRequiredMixin, PermissionRequiredMixin, View):
|
|||
template_name = "ui/insights/insights.html"
|
||||
permission_required = "use_insights"
|
||||
|
||||
def get(self, request):
|
||||
return render(request, self.template_name)
|
||||
def get(self, request, index):
|
||||
context = {"index": index}
|
||||
return render(request, self.template_name, context)
|
||||
|
||||
|
||||
class InsightsSearch(LoginRequiredMixin, PermissionRequiredMixin, View):
|
||||
|
@ -32,13 +33,16 @@ class InsightsSearch(LoginRequiredMixin, PermissionRequiredMixin, View):
|
|||
template_name = "ui/insights/info.html"
|
||||
permission_required = "use_insights"
|
||||
|
||||
def post(self, request):
|
||||
def post(self, request, index):
|
||||
query_params = request.POST.dict()
|
||||
if "query_full" in query_params:
|
||||
query_params["query_full"] = "nick: " + query_params["query_full"]
|
||||
context = query_single_result(request, query_params)
|
||||
if "query" in query_params:
|
||||
query_params["query"] = "nick: " + query_params["query"]
|
||||
query_params["source"] = "all"
|
||||
query_params["index"] = index
|
||||
context = db.query_single_result(request, query_params)
|
||||
if not context:
|
||||
return HttpResponseForbidden()
|
||||
context["index"] = index
|
||||
return render(request, self.template_name, context)
|
||||
|
||||
|
||||
|
@ -47,7 +51,7 @@ class InsightsChannels(LoginRequiredMixin, PermissionRequiredMixin, APIView):
|
|||
template_name = "ui/insights/channels.html"
|
||||
permission_required = "use_insights"
|
||||
|
||||
def post(self, request):
|
||||
def post(self, request, index):
|
||||
if "net" not in request.data:
|
||||
return HttpResponse("No net")
|
||||
if "nick" not in request.data:
|
||||
|
@ -58,7 +62,13 @@ class InsightsChannels(LoginRequiredMixin, PermissionRequiredMixin, APIView):
|
|||
num_users = annotate_num_users(net, chans)
|
||||
if not chans:
|
||||
return HttpResponseForbidden()
|
||||
context = {"net": net, "nick": nick, "chans": chans, "num_users": num_users}
|
||||
context = {
|
||||
"net": net,
|
||||
"nick": nick,
|
||||
"chans": chans,
|
||||
"num_users": num_users,
|
||||
"index": index,
|
||||
}
|
||||
return render(request, self.template_name, context)
|
||||
|
||||
|
||||
|
@ -67,7 +77,7 @@ class InsightsNicks(LoginRequiredMixin, PermissionRequiredMixin, APIView):
|
|||
template_name = "ui/insights/nicks.html"
|
||||
permission_required = "use_insights"
|
||||
|
||||
def post(self, request):
|
||||
def post(self, request, index):
|
||||
if "net" not in request.data:
|
||||
return HttpResponse("No net")
|
||||
if "nick" not in request.data:
|
||||
|
@ -82,7 +92,13 @@ class InsightsNicks(LoginRequiredMixin, PermissionRequiredMixin, APIView):
|
|||
online = annotate_online(net, nicks)
|
||||
if not nicks:
|
||||
return HttpResponseForbidden()
|
||||
context = {"net": net, "nick": nick, "nicks": nicks, "online": online}
|
||||
context = {
|
||||
"net": net,
|
||||
"nick": nick,
|
||||
"nicks": nicks,
|
||||
"online": online,
|
||||
"index": index,
|
||||
}
|
||||
return render(request, self.template_name, context)
|
||||
|
||||
|
||||
|
@ -91,7 +107,7 @@ class InsightsMeta(LoginRequiredMixin, PermissionRequiredMixin, APIView):
|
|||
template_name = "ui/insights/meta.html"
|
||||
permission_required = "use_insights"
|
||||
|
||||
def post(self, request):
|
||||
def post(self, request, index):
|
||||
if "net" not in request.data:
|
||||
return HttpResponse("No net")
|
||||
if "nicks" not in request.data:
|
||||
|
@ -99,6 +115,10 @@ class InsightsMeta(LoginRequiredMixin, PermissionRequiredMixin, APIView):
|
|||
net = request.data["net"]
|
||||
nicks = request.data["nicks"]
|
||||
nicks = literal_eval(nicks)
|
||||
|
||||
# Check the user has permissions to use the meta index
|
||||
if not request.user.has_perm("core.index_meta"):
|
||||
return HttpResponseForbidden()
|
||||
meta = get_meta(request, net, nicks)
|
||||
unique_values = {}
|
||||
# Create a map of unique values for each key for each nick
|
||||
|
@ -122,7 +142,7 @@ class InsightsMeta(LoginRequiredMixin, PermissionRequiredMixin, APIView):
|
|||
meta_dedup[k].add(v)
|
||||
unique_values[nick][k].remove(v)
|
||||
|
||||
context = {"net": net, "nicks": nicks, "meta": meta_dedup}
|
||||
context = {"net": net, "nicks": nicks, "meta": meta_dedup, "index": index}
|
||||
return render(request, self.template_name, context)
|
||||
|
||||
|
||||
|
@ -131,7 +151,7 @@ class InsightsInfoModal(LoginRequiredMixin, PermissionRequiredMixin, APIView):
|
|||
template_name = "modals/drilldown.html"
|
||||
permission_required = "use_insights"
|
||||
|
||||
def post(self, request):
|
||||
def post(self, request, index):
|
||||
if "net" not in request.data:
|
||||
return JsonResponse({"success": False})
|
||||
if "nick" not in request.data:
|
||||
|
@ -163,5 +183,6 @@ class InsightsInfoModal(LoginRequiredMixin, PermissionRequiredMixin, APIView):
|
|||
"inter_users": inter_users,
|
||||
"num_users": num_users,
|
||||
"num_chans": num_chans,
|
||||
"index": index,
|
||||
}
|
||||
return render(request, self.template_name, context)
|
||||
|
|
Loading…
Reference in New Issue