From 185bda02eaaadb485f989b11b7163d387e208617 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 21 Jul 2022 13:51:55 +0100 Subject: [PATCH] Implement Insights page --- app/urls.py | 24 ++- core/api/views/threshold.py | 52 +---- core/lib/opensearch.py | 204 ++++++++++++++---- core/templates/base.html | 5 +- .../modals/{info.html => drilldown.html} | 0 core/templates/modals/insights.html | 132 ++++++++++++ .../ui/{ => drilldown}/drilldown.html | 39 +--- .../templates/ui/{ => drilldown}/results.html | 3 +- core/templates/ui/insights/insights.html | 85 ++++++++ core/templates/ui/insights/results.html | 79 +++++++ core/ui/views/drilldown.py | 8 +- core/ui/views/insights.py | 13 ++ core/views/dynamic/drilldown.py | 88 ++++++++ core/views/dynamic/insights.py | 75 +++++++ core/views/dynamic/search.py | 119 ---------- 15 files changed, 652 insertions(+), 274 deletions(-) rename core/templates/modals/{info.html => drilldown.html} (100%) create mode 100644 core/templates/modals/insights.html rename core/templates/ui/{ => drilldown}/drilldown.html (63%) rename core/templates/ui/{ => drilldown}/results.html (98%) create mode 100644 core/templates/ui/insights/insights.html create mode 100644 core/templates/ui/insights/results.html create mode 100644 core/ui/views/insights.py create mode 100644 core/views/dynamic/drilldown.py create mode 100644 core/views/dynamic/insights.py delete mode 100644 core/views/dynamic/search.py diff --git a/app/urls.py b/app/urls.py index b1fc77f..18ddfec 100644 --- a/app/urls.py +++ b/app/urls.py @@ -19,16 +19,18 @@ from django.contrib import admin from django.urls import include, path from django.views.generic import TemplateView -from core.api.views.threshold import ( - ThresholdChans, - ThresholdInfoModal, - ThresholdOnline, - ThresholdUsers, -) +# Threshold API stuff +from core.api.views.threshold import ThresholdChans, ThresholdOnline, ThresholdUsers + +# Main tool pages from core.ui.views.drilldown import Drilldown +from core.ui.views.insights import Insights from core.views import Billing, Cancel, Home, Order, Portal, Signup from core.views.callbacks import Callback -from core.views.dynamic.search import APISearch, Search + +# Dynamic elements +from core.views.dynamic.drilldown import DrilldownSearch, ThresholdInfoModal +from core.views.dynamic.insights import InsightsInfoModal, InsightsSearch urlpatterns = [ path("", Home.as_view(), name="home"), @@ -49,10 +51,12 @@ urlpatterns = [ path("accounts/", include("django.contrib.auth.urls")), path("accounts/signup/", Signup.as_view(), name="signup"), path("ui/drilldown/", Drilldown.as_view(), name="drilldown"), - path("parts/search/", Search.as_view(), name="search"), - path("modal/info/", ThresholdInfoModal.as_view(), name="modal_info"), + path("ui/insights/", Insights.as_view(), name="insights"), + path("parts/search/drilldown/", DrilldownSearch.as_view(), name="search_drilldown"), + path("parts/search/insights/", InsightsSearch.as_view(), name="search_insights"), + path("modal/drilldown/", ThresholdInfoModal.as_view(), name="modal_drilldown"), + path("modal/insights/", InsightsInfoModal.as_view(), name="modal_insights"), path("api/chans/", ThresholdChans.as_view(), name="chans"), path("api/users/", ThresholdUsers.as_view(), name="users"), path("api/online/", ThresholdOnline.as_view(), name="online"), - path("api/search/", APISearch.as_view(), name="api_search"), ] + static(settings.STATIC_URL, document_root=settings.STATIC_ROOT) diff --git a/core/api/views/threshold.py b/core/api/views/threshold.py index fc7b508..d4453d7 100644 --- a/core/api/views/threshold.py +++ b/core/api/views/threshold.py @@ -2,17 +2,10 @@ import logging from django.contrib.auth.mixins import LoginRequiredMixin from django.http import HttpResponse, JsonResponse -from django.shortcuts import render from rest_framework.parsers import FormParser from rest_framework.views import APIView -from core.lib.threshold import ( - annotate_num_chans, - annotate_num_users, - annotate_online, - get_chans, - get_users, -) +from core.lib.threshold import annotate_online, get_chans, get_users logger = logging.getLogger(__name__) @@ -72,46 +65,3 @@ class ThresholdOnline(LoginRequiredMixin, APIView): query = request.data["query"] online_info = annotate_online(net, query) return JsonResponse(online_info) - - -class ThresholdInfoModal(LoginRequiredMixin, APIView): - parser_classes = [FormParser] - plan_name = "drilldown" - template_name = "modals/info.html" - - def post(self, request): - if not request.user.has_plan(self.plan_name): - return JsonResponse({"success": False}) - if "net" not in request.data: - return JsonResponse({"success": False}) - if "nick" not in request.data: - return JsonResponse({"success": False}) - if "channel" not in request.data: - return JsonResponse({"success": False}) - net = request.data["net"] - nick = request.data["nick"] - channel = request.data["channel"] - channels = get_chans(net, [nick]) - users = get_users(net, [channel]) - num_users = annotate_num_users(net, channels) - num_chans = annotate_num_chans(net, users) - if channels: - inter_users = get_users(net, channels) - else: - inter_users = [] - if users: - inter_chans = get_chans(net, users) - else: - inter_chans = [] - context = { - "net": net, - "nick": nick, - "channel": channel, - "chans": channels, - "users": users, - "inter_chans": inter_chans, - "inter_users": inter_users, - "num_users": num_users, - "num_chans": num_chans, - } - return render(request, self.template_name, context) diff --git a/core/lib/opensearch.py b/core/lib/opensearch.py index f084dc2..5829e7b 100644 --- a/core/lib/opensearch.py +++ b/core/lib/opensearch.py @@ -2,8 +2,13 @@ from django.conf import settings from opensearchpy import OpenSearch from opensearchpy.exceptions import RequestError +from core.lib.threshold import annotate_num_chans, annotate_num_users, annotate_online + def initialise_opensearch(): + """ + Inititialise the OpenSearch API endpoint. + """ auth = (settings.OPENSEARCH_USERNAME, settings.OPENSEARCH_PASSWORD) client = OpenSearch( # fmt: off @@ -22,9 +27,157 @@ def initialise_opensearch(): return client -def construct_query(query, fields, size): - if not fields: - fields = settings.OPENSEARCH_MAIN_SEARCH_FIELDS +client = initialise_opensearch() + + +def annotate_results(results_parsed): + """ + Accept a list of dict objects, search for the number of channels and users. + Add them to the object. + Mutate it in place. Does not return anything. + """ + # Figure out items with net (not discord) + nets = set() + for x in results_parsed: + if "net" in x: + nets.add(x["net"]) + + for net in nets: + # Annotate the online attribute from Threshold + online_info = annotate_online( + net, [x["nick"] for x in results_parsed if x["src"] == "irc"] + ) + # Annotate the number of users in the channel + num_users = annotate_num_users( + net, [x["channel"] for x in results_parsed if x["src"] == "irc"] + ) + # Annotate the number channels the user is on + num_chans = annotate_num_chans( + net, [x["nick"] for x in results_parsed if x["src"] == "irc"] + ) + for item in results_parsed: + if item["nick"] in online_info: + item["online"] = online_info[item["nick"]] + if item["channel"] in num_users: + item["num_users"] = num_users[item["channel"]] + if item["nick"] in num_chans: + item["num_chans"] = num_chans[item["nick"]] + + +def filter_blacklisted(user, response): + """ + Low level filter to take the raw OpenSearch response and remove + objects from it we want to keep secret. + Does not return, the object is mutated in place. + """ + response["redacted"] = 0 + response["exemption"] = None + # For every hit from ES + for item in list(response["hits"]["hits"]): + # For every blacklisted type + for blacklisted_type in settings.OPENSEARCH_BLACKLISTED.keys(): + # Check this field we are matching exists + if blacklisted_type in item["_source"].keys(): + content = item["_source"][blacklisted_type] + # For every item in the blacklisted array for the type + for blacklisted_item in settings.OPENSEARCH_BLACKLISTED[ + blacklisted_type + ]: + if blacklisted_item in str(content): + # Remove the item + if item in response["hits"]["hits"]: + if not user.is_superuser: + response["hits"]["hits"].remove(item) + # Let the UI know something was redacted + response["redacted"] += 1 + response["exemption"] = True + + +def run_main_query(client, user, query, size=None): + """ + Low level helper to run an ES query. + Accept a user to pass it to the filter, so we can + avoid filtering for superusers. + Accept fields and size, for the fields we want to match and the + number of results to return. + """ + search_query = construct_query(query, size) + try: + response = client.search( + body=search_query, index=settings.OPENSEARCH_INDEX_MAIN + ) + except RequestError: + print("REQUEST ERROR") + return False + filter_blacklisted(user, response) + return response + + +def query_results(request, size=None): + """ + API helper to alter the OpenSearch return format into something + a bit better to parse. + Accept a HTTP request object. Run the query, and annotate the + results with the other data we have. + """ + if not size: + if "size" in request.POST: + size = request.POST["size"] + if size not in settings.OPENSEARCH_MAIN_SIZES: + return False + if "query" in request.POST: + query = request.POST["query"] + results = run_main_query( + client, + request.user, + query, + size, + ) + if not results: + return False + results_parsed = [] + if "hits" in results.keys(): + if "hits" in results["hits"]: + for item in results["hits"]["hits"]: + element = item["_source"] + element["id"] = item["_id"] + + # Split the timestamp into date and time + ts = element["ts"] + ts_spl = ts.split("T") + date = ts_spl[0] + time = ts_spl[1] + element["date"] = date + element["time"] = time + results_parsed.append(element) + + annotate_results(results_parsed) + + context = { + "query": query, + "results": results_parsed, + "card": results["hits"]["total"]["value"], + "took": results["took"], + "redacted": results["redacted"], + "exemption": results["exemption"], + } + return context + + +def query_single_result(request): + context = query_results(request, 1) + dedup_set = {item["nick"] for item in context["results"]} + if len(dedup_set) == 1: + context["item"] = context["results"][0] + else: + return (len(dedup_set), context) + return (1, context) + + +def construct_query(query, size): + """ + Accept some query parameters and construct an OpenSearch query. + """ if not size: size = 5 query = { @@ -32,7 +185,7 @@ def construct_query(query, fields, size): "query": { "query_string": { "query": query, - "fields": fields, + # "fields": fields, # "default_field": "msg", # "type": "best_fields", "fuzziness": "AUTO", @@ -63,46 +216,3 @@ def construct_query(query, fields, size): ], } return query - - -def filter_blacklisted(user, response): - response["redacted"] = 0 - response["exemption"] = None - # For every hit from ES - for item in list(response["hits"]["hits"]): - # For every blacklisted type - for blacklisted_type in settings.OPENSEARCH_BLACKLISTED.keys(): - # Check this field we are matching exists - if blacklisted_type in item["_source"].keys(): - content = item["_source"][blacklisted_type] - # For every item in the blacklisted array for the type - for blacklisted_item in settings.OPENSEARCH_BLACKLISTED[ - blacklisted_type - ]: - if blacklisted_item in str(content): - # Remove the item - if item in response["hits"]["hits"]: - if not user.is_superuser: - response["hits"]["hits"].remove(item) - # Let the UI know something was redacted - response["redacted"] += 1 - response["exemption"] = True - - -def run_main_query(client, user, query, fields=None, size=None): - if fields: - for field in fields: - if field not in settings.OPENSEARCH_MAIN_SEARCH_FIELDS: - return False - if size: - if size not in settings.OPENSEARCH_MAIN_SIZES: - return False - search_query = construct_query(query, fields, size) - # fmt: off - try: - response = client.search(body=search_query, - index=settings.OPENSEARCH_INDEX_MAIN) - except RequestError: - return False - filter_blacklisted(user, response) - return response diff --git a/core/templates/base.html b/core/templates/base.html index 9c21169..5fda1ee 100644 --- a/core/templates/base.html +++ b/core/templates/base.html @@ -65,7 +65,7 @@ {% if user.is_authenticated %}