diff --git a/app/urls.py b/app/urls.py index b1fc77f..18ddfec 100644 --- a/app/urls.py +++ b/app/urls.py @@ -19,16 +19,18 @@ from django.contrib import admin from django.urls import include, path from django.views.generic import TemplateView -from core.api.views.threshold import ( - ThresholdChans, - ThresholdInfoModal, - ThresholdOnline, - ThresholdUsers, -) +# Threshold API stuff +from core.api.views.threshold import ThresholdChans, ThresholdOnline, ThresholdUsers + +# Main tool pages from core.ui.views.drilldown import Drilldown +from core.ui.views.insights import Insights from core.views import Billing, Cancel, Home, Order, Portal, Signup from core.views.callbacks import Callback -from core.views.dynamic.search import APISearch, Search + +# Dynamic elements +from core.views.dynamic.drilldown import DrilldownSearch, ThresholdInfoModal +from core.views.dynamic.insights import InsightsInfoModal, InsightsSearch urlpatterns = [ path("", Home.as_view(), name="home"), @@ -49,10 +51,12 @@ urlpatterns = [ path("accounts/", include("django.contrib.auth.urls")), path("accounts/signup/", Signup.as_view(), name="signup"), path("ui/drilldown/", Drilldown.as_view(), name="drilldown"), - path("parts/search/", Search.as_view(), name="search"), - path("modal/info/", ThresholdInfoModal.as_view(), name="modal_info"), + path("ui/insights/", Insights.as_view(), name="insights"), + path("parts/search/drilldown/", DrilldownSearch.as_view(), name="search_drilldown"), + path("parts/search/insights/", InsightsSearch.as_view(), name="search_insights"), + path("modal/drilldown/", ThresholdInfoModal.as_view(), name="modal_drilldown"), + path("modal/insights/", InsightsInfoModal.as_view(), name="modal_insights"), path("api/chans/", ThresholdChans.as_view(), name="chans"), path("api/users/", ThresholdUsers.as_view(), name="users"), path("api/online/", ThresholdOnline.as_view(), name="online"), - path("api/search/", APISearch.as_view(), name="api_search"), ] + static(settings.STATIC_URL, document_root=settings.STATIC_ROOT) diff --git a/core/api/views/threshold.py b/core/api/views/threshold.py index fc7b508..d4453d7 100644 --- a/core/api/views/threshold.py +++ b/core/api/views/threshold.py @@ -2,17 +2,10 @@ import logging from django.contrib.auth.mixins import LoginRequiredMixin from django.http import HttpResponse, JsonResponse -from django.shortcuts import render from rest_framework.parsers import FormParser from rest_framework.views import APIView -from core.lib.threshold import ( - annotate_num_chans, - annotate_num_users, - annotate_online, - get_chans, - get_users, -) +from core.lib.threshold import annotate_online, get_chans, get_users logger = logging.getLogger(__name__) @@ -72,46 +65,3 @@ class ThresholdOnline(LoginRequiredMixin, APIView): query = request.data["query"] online_info = annotate_online(net, query) return JsonResponse(online_info) - - -class ThresholdInfoModal(LoginRequiredMixin, APIView): - parser_classes = [FormParser] - plan_name = "drilldown" - template_name = "modals/info.html" - - def post(self, request): - if not request.user.has_plan(self.plan_name): - return JsonResponse({"success": False}) - if "net" not in request.data: - return JsonResponse({"success": False}) - if "nick" not in request.data: - return JsonResponse({"success": False}) - if "channel" not in request.data: - return JsonResponse({"success": False}) - net = request.data["net"] - nick = request.data["nick"] - channel = request.data["channel"] - channels = get_chans(net, [nick]) - users = get_users(net, [channel]) - num_users = annotate_num_users(net, channels) - num_chans = annotate_num_chans(net, users) - if channels: - inter_users = get_users(net, channels) - else: - inter_users = [] - if users: - inter_chans = get_chans(net, users) - else: - inter_chans = [] - context = { - "net": net, - "nick": nick, - "channel": channel, - "chans": channels, - "users": users, - "inter_chans": inter_chans, - "inter_users": inter_users, - "num_users": num_users, - "num_chans": num_chans, - } - return render(request, self.template_name, context) diff --git a/core/lib/opensearch.py b/core/lib/opensearch.py index f084dc2..5829e7b 100644 --- a/core/lib/opensearch.py +++ b/core/lib/opensearch.py @@ -2,8 +2,13 @@ from django.conf import settings from opensearchpy import OpenSearch from opensearchpy.exceptions import RequestError +from core.lib.threshold import annotate_num_chans, annotate_num_users, annotate_online + def initialise_opensearch(): + """ + Inititialise the OpenSearch API endpoint. + """ auth = (settings.OPENSEARCH_USERNAME, settings.OPENSEARCH_PASSWORD) client = OpenSearch( # fmt: off @@ -22,9 +27,157 @@ def initialise_opensearch(): return client -def construct_query(query, fields, size): - if not fields: - fields = settings.OPENSEARCH_MAIN_SEARCH_FIELDS +client = initialise_opensearch() + + +def annotate_results(results_parsed): + """ + Accept a list of dict objects, search for the number of channels and users. + Add them to the object. + Mutate it in place. Does not return anything. + """ + # Figure out items with net (not discord) + nets = set() + for x in results_parsed: + if "net" in x: + nets.add(x["net"]) + + for net in nets: + # Annotate the online attribute from Threshold + online_info = annotate_online( + net, [x["nick"] for x in results_parsed if x["src"] == "irc"] + ) + # Annotate the number of users in the channel + num_users = annotate_num_users( + net, [x["channel"] for x in results_parsed if x["src"] == "irc"] + ) + # Annotate the number channels the user is on + num_chans = annotate_num_chans( + net, [x["nick"] for x in results_parsed if x["src"] == "irc"] + ) + for item in results_parsed: + if item["nick"] in online_info: + item["online"] = online_info[item["nick"]] + if item["channel"] in num_users: + item["num_users"] = num_users[item["channel"]] + if item["nick"] in num_chans: + item["num_chans"] = num_chans[item["nick"]] + + +def filter_blacklisted(user, response): + """ + Low level filter to take the raw OpenSearch response and remove + objects from it we want to keep secret. + Does not return, the object is mutated in place. + """ + response["redacted"] = 0 + response["exemption"] = None + # For every hit from ES + for item in list(response["hits"]["hits"]): + # For every blacklisted type + for blacklisted_type in settings.OPENSEARCH_BLACKLISTED.keys(): + # Check this field we are matching exists + if blacklisted_type in item["_source"].keys(): + content = item["_source"][blacklisted_type] + # For every item in the blacklisted array for the type + for blacklisted_item in settings.OPENSEARCH_BLACKLISTED[ + blacklisted_type + ]: + if blacklisted_item in str(content): + # Remove the item + if item in response["hits"]["hits"]: + if not user.is_superuser: + response["hits"]["hits"].remove(item) + # Let the UI know something was redacted + response["redacted"] += 1 + response["exemption"] = True + + +def run_main_query(client, user, query, size=None): + """ + Low level helper to run an ES query. + Accept a user to pass it to the filter, so we can + avoid filtering for superusers. + Accept fields and size, for the fields we want to match and the + number of results to return. + """ + search_query = construct_query(query, size) + try: + response = client.search( + body=search_query, index=settings.OPENSEARCH_INDEX_MAIN + ) + except RequestError: + print("REQUEST ERROR") + return False + filter_blacklisted(user, response) + return response + + +def query_results(request, size=None): + """ + API helper to alter the OpenSearch return format into something + a bit better to parse. + Accept a HTTP request object. Run the query, and annotate the + results with the other data we have. + """ + if not size: + if "size" in request.POST: + size = request.POST["size"] + if size not in settings.OPENSEARCH_MAIN_SIZES: + return False + if "query" in request.POST: + query = request.POST["query"] + results = run_main_query( + client, + request.user, + query, + size, + ) + if not results: + return False + results_parsed = [] + if "hits" in results.keys(): + if "hits" in results["hits"]: + for item in results["hits"]["hits"]: + element = item["_source"] + element["id"] = item["_id"] + + # Split the timestamp into date and time + ts = element["ts"] + ts_spl = ts.split("T") + date = ts_spl[0] + time = ts_spl[1] + element["date"] = date + element["time"] = time + results_parsed.append(element) + + annotate_results(results_parsed) + + context = { + "query": query, + "results": results_parsed, + "card": results["hits"]["total"]["value"], + "took": results["took"], + "redacted": results["redacted"], + "exemption": results["exemption"], + } + return context + + +def query_single_result(request): + context = query_results(request, 1) + dedup_set = {item["nick"] for item in context["results"]} + if len(dedup_set) == 1: + context["item"] = context["results"][0] + else: + return (len(dedup_set), context) + return (1, context) + + +def construct_query(query, size): + """ + Accept some query parameters and construct an OpenSearch query. + """ if not size: size = 5 query = { @@ -32,7 +185,7 @@ def construct_query(query, fields, size): "query": { "query_string": { "query": query, - "fields": fields, + # "fields": fields, # "default_field": "msg", # "type": "best_fields", "fuzziness": "AUTO", @@ -63,46 +216,3 @@ def construct_query(query, fields, size): ], } return query - - -def filter_blacklisted(user, response): - response["redacted"] = 0 - response["exemption"] = None - # For every hit from ES - for item in list(response["hits"]["hits"]): - # For every blacklisted type - for blacklisted_type in settings.OPENSEARCH_BLACKLISTED.keys(): - # Check this field we are matching exists - if blacklisted_type in item["_source"].keys(): - content = item["_source"][blacklisted_type] - # For every item in the blacklisted array for the type - for blacklisted_item in settings.OPENSEARCH_BLACKLISTED[ - blacklisted_type - ]: - if blacklisted_item in str(content): - # Remove the item - if item in response["hits"]["hits"]: - if not user.is_superuser: - response["hits"]["hits"].remove(item) - # Let the UI know something was redacted - response["redacted"] += 1 - response["exemption"] = True - - -def run_main_query(client, user, query, fields=None, size=None): - if fields: - for field in fields: - if field not in settings.OPENSEARCH_MAIN_SEARCH_FIELDS: - return False - if size: - if size not in settings.OPENSEARCH_MAIN_SIZES: - return False - search_query = construct_query(query, fields, size) - # fmt: off - try: - response = client.search(body=search_query, - index=settings.OPENSEARCH_INDEX_MAIN) - except RequestError: - return False - filter_blacklisted(user, response) - return response diff --git a/core/templates/base.html b/core/templates/base.html index 9c21169..5fda1ee 100644 --- a/core/templates/base.html +++ b/core/templates/base.html @@ -65,7 +65,7 @@ {% if user.is_authenticated %}
- Plans + Tools
@@ -73,6 +73,9 @@ Drilldown + + Insights + {% endif %}
diff --git a/core/templates/modals/info.html b/core/templates/modals/drilldown.html similarity index 100% rename from core/templates/modals/info.html rename to core/templates/modals/drilldown.html diff --git a/core/templates/modals/insights.html b/core/templates/modals/insights.html new file mode 100644 index 0000000..9d7587d --- /dev/null +++ b/core/templates/modals/insights.html @@ -0,0 +1,132 @@ +{% load index %} + + + +
+
+
+
+ +
+
+

Channels for {{ nick }} on {{ net }}

+ {% for channel in chans %} + + + + + {{ channel }} + {% if nick in num_chans %} + + {{ num_users|index:channel }} + + {% endif %} + + {% endfor %} +
+
+

Users on {{ channel }} for {{ net }}

+ {% for user in users %} + + + + + {{ user }} + {% if channel in num_users %} + + {{ num_chans|index:user }} + + {% endif %} + + {% endfor %} +
+
+

Users sharing channels with {{ nick }} on {{ net }}

+ {% for user in inter_users %} + + + + + {{ user }} + {% if channel in num_users %} + + {{ num_chans|index:user }} + + {% endif %} + + {% endfor %} +
+
+

Channels sharing users with {{ channel }} on {{ net }}

+ {% for channel in inter_chans %} + + + + + {{ channel }} + {% if nick in num_chans %} + + {{ num_users|index:channel }} + + {% endif %} + + {% endfor %} +
+
+
+ +
+
diff --git a/core/templates/ui/drilldown.html b/core/templates/ui/drilldown/drilldown.html similarity index 63% rename from core/templates/ui/drilldown.html rename to core/templates/ui/drilldown/drilldown.html index a880bf5..167e870 100644 --- a/core/templates/ui/drilldown.html +++ b/core/templates/ui/drilldown/drilldown.html @@ -56,43 +56,6 @@
-
- -
-
-
-
- - - - -
-
-
-
-
-
- - -
-
-
- - - - -
-
-
-
@@ -114,7 +77,7 @@
-
diff --git a/core/templates/ui/results.html b/core/templates/ui/drilldown/results.html similarity index 98% rename from core/templates/ui/results.html rename to core/templates/ui/drilldown/results.html index 32b2e4e..75f3af6 100644 --- a/core/templates/ui/results.html +++ b/core/templates/ui/drilldown/results.html @@ -1,4 +1,5 @@ {% load static %} +{% load index %}
{% if results is not None %} @@ -106,7 +107,7 @@ {% if item.src == 'irc' %} +
+
+ +
+ +
+ +
+ +
+ +
+

2

+
+ +
+ + + + + +{% endblock %} diff --git a/core/templates/ui/insights/results.html b/core/templates/ui/insights/results.html new file mode 100644 index 0000000..0a87e53 --- /dev/null +++ b/core/templates/ui/insights/results.html @@ -0,0 +1,79 @@ +{% load static %} +{% load index %} + +
+
+ {% if item is not None %} +
+ + + + + + + + + + + + + + + + + + + + + + +
src + {% if item|index:'src' == 'irc' %} + + + + {% elif item|index:'src' == 'dis' %} + + + + {% endif %} +
nick + {% if item|index:'online' is True %} + + + + {{ item.nick }} + {% elif item|index:'online' is False %} + + + + {{ item|index:'nick' }} + {% else %} + + + + {{ item|index:'nick'}} + {% endif %} + {% if item|index:'num_chans' is not None %} + + {{ item|index:'num_chans' }} + + {% endif %} +
host{{ item.host }}
actions + {% if item.src == 'irc' %} + +
+ {% endif %} +
net{{ item.net }}
+
+ {% endif %} +
+
\ No newline at end of file diff --git a/core/ui/views/drilldown.py b/core/ui/views/drilldown.py index 9bef317..686ce50 100644 --- a/core/ui/views/drilldown.py +++ b/core/ui/views/drilldown.py @@ -3,21 +3,15 @@ from django.contrib.auth.mixins import LoginRequiredMixin from django.shortcuts import render from django.views import View -from core.lib.opensearch import initialise_opensearch - -client = initialise_opensearch() - class Drilldown(LoginRequiredMixin, View): - template_name = "ui/drilldown.html" + template_name = "ui/drilldown/drilldown.html" plan_name = "drilldown" def get(self, request): if not request.user.has_plan(self.plan_name): return render(request, "denied.html") context = { - "fields": settings.OPENSEARCH_MAIN_SEARCH_FIELDS, "sizes": settings.OPENSEARCH_MAIN_SIZES, - "timescales": settings.OPENSEARCH_MAIN_TIMESCALES, } return render(request, self.template_name, context) diff --git a/core/ui/views/insights.py b/core/ui/views/insights.py new file mode 100644 index 0000000..c4cc23e --- /dev/null +++ b/core/ui/views/insights.py @@ -0,0 +1,13 @@ +from django.contrib.auth.mixins import LoginRequiredMixin +from django.shortcuts import render +from django.views import View + + +class Insights(LoginRequiredMixin, View): + template_name = "ui/insights/insights.html" + plan_name = "drilldown" + + def get(self, request): + if not request.user.has_plan(self.plan_name): + return render(request, "denied.html") + return render(request, self.template_name) diff --git a/core/views/dynamic/drilldown.py b/core/views/dynamic/drilldown.py new file mode 100644 index 0000000..41d3765 --- /dev/null +++ b/core/views/dynamic/drilldown.py @@ -0,0 +1,88 @@ +import json + +from django.contrib.auth.mixins import LoginRequiredMixin +from django.http import HttpResponse, HttpResponseForbidden, JsonResponse +from django.shortcuts import render +from django.views import View +from rest_framework.parsers import FormParser +from rest_framework.views import APIView + +from core.lib.opensearch import query_results +from core.lib.threshold import ( + annotate_num_chans, + annotate_num_users, + get_chans, + get_users, +) + + +class DrilldownSearch(LoginRequiredMixin, View): + # parser_classes = [JSONParser] + template_name = "ui/drilldown/results.html" + plan_name = "drilldown" + + def post(self, request): + if not request.user.has_plan(self.plan_name): + return HttpResponseForbidden() + + context = query_results(request) + if not context: + return HttpResponseForbidden() + context["data"] = json.dumps( + [ + { + "text": item.get("msg", None) or item.get("id"), + "nick": item.get("nick", None), + "value": item.get("sentiment", None) or None, + "date": item.get("ts"), + } + for item in context["results"] + ] + ) + if context: + return render(request, self.template_name, context) + else: + return HttpResponse("No results") + + +class ThresholdInfoModal(LoginRequiredMixin, APIView): + parser_classes = [FormParser] + plan_name = "drilldown" + template_name = "modals/drilldown.html" + + def post(self, request): + if not request.user.has_plan(self.plan_name): + return JsonResponse({"success": False}) + if "net" not in request.data: + return JsonResponse({"success": False}) + if "nick" not in request.data: + return JsonResponse({"success": False}) + if "channel" not in request.data: + return JsonResponse({"success": False}) + net = request.data["net"] + nick = request.data["nick"] + channel = request.data["channel"] + channels = get_chans(net, [nick]) + users = get_users(net, [channel]) + num_users = annotate_num_users(net, channels) + num_chans = annotate_num_chans(net, users) + if channels: + inter_users = get_users(net, channels) + else: + inter_users = [] + if users: + inter_chans = get_chans(net, users) + else: + inter_chans = [] + context = { + "net": net, + "nick": nick, + "channel": channel, + "chans": channels, + "users": users, + "inter_chans": inter_chans, + "inter_users": inter_users, + "num_users": num_users, + "num_chans": num_chans, + } + return render(request, self.template_name, context) diff --git a/core/views/dynamic/insights.py b/core/views/dynamic/insights.py new file mode 100644 index 0000000..20cf62a --- /dev/null +++ b/core/views/dynamic/insights.py @@ -0,0 +1,75 @@ +from django.contrib.auth.mixins import LoginRequiredMixin +from django.http import HttpResponse, HttpResponseForbidden, JsonResponse +from django.shortcuts import render +from django.views import View +from rest_framework.parsers import FormParser +from rest_framework.views import APIView + +from core.lib.opensearch import query_single_result +from core.lib.threshold import ( + annotate_num_chans, + annotate_num_users, + get_chans, + get_users, +) + + +class InsightsSearch(LoginRequiredMixin, View): + # parser_classes = [JSONParser] + template_name = "ui/insights/results.html" + plan_name = "drilldown" + + def post(self, request): + if not request.user.has_plan(self.plan_name): + return HttpResponseForbidden() + + results, context = query_single_result(request) + if not context: + return HttpResponseForbidden() + if context: + return render(request, self.template_name, context) + else: + return HttpResponse("No results") + + +class InsightsInfoModal(LoginRequiredMixin, APIView): + parser_classes = [FormParser] + plan_name = "drilldown" + template_name = "modals/insights.html" + + def post(self, request): + if not request.user.has_plan(self.plan_name): + return JsonResponse({"success": False}) + if "net" not in request.data: + return JsonResponse({"success": False}) + if "nick" not in request.data: + return JsonResponse({"success": False}) + if "channel" not in request.data: + return JsonResponse({"success": False}) + net = request.data["net"] + nick = request.data["nick"] + channel = request.data["channel"] + channels = get_chans(net, [nick]) + users = get_users(net, [channel]) + num_users = annotate_num_users(net, channels) + num_chans = annotate_num_chans(net, users) + if channels: + inter_users = get_users(net, channels) + else: + inter_users = [] + if users: + inter_chans = get_chans(net, users) + else: + inter_chans = [] + context = { + "net": net, + "nick": nick, + "channel": channel, + "chans": channels, + "users": users, + "inter_chans": inter_chans, + "inter_users": inter_users, + "num_users": num_users, + "num_chans": num_chans, + } + return render(request, self.template_name, context) diff --git a/core/views/dynamic/search.py b/core/views/dynamic/search.py deleted file mode 100644 index 71bbd27..0000000 --- a/core/views/dynamic/search.py +++ /dev/null @@ -1,119 +0,0 @@ -import json - -from django.conf import settings -from django.contrib.auth.mixins import LoginRequiredMixin -from django.http import HttpResponse, HttpResponseForbidden, JsonResponse -from django.shortcuts import render -from django.views import View - -from core.lib.opensearch import initialise_opensearch, run_main_query -from core.lib.threshold import annotate_num_chans, annotate_num_users, annotate_online - -client = initialise_opensearch() - - -def query_results(request, post_params, api=False): - fields = None - if "fields" in request.POST: - fields = request.POST.getlist("fields") - if "size" in request.POST: - size = request.POST["size"] - if "query" in request.POST: - query = request.POST["query"] - results = run_main_query(client, request.user, query, fields, size) - if not results: - return False - results_parsed = [] - if "hits" in results.keys(): - if "hits" in results["hits"]: - for item in results["hits"]["hits"]: - element = item["_source"] - element["id"] = item["_id"] - ts = element["ts"] - ts_spl = ts.split("T") - date = ts_spl[0] - time = ts_spl[1] - element["date"] = date - element["time"] = time - results_parsed.append(element) - # Figure out items with net (not discord) - nets = set() - for x in results_parsed: - if "net" in x: - nets.add(x["net"]) - - for net in nets: - # Annotate the online attribute from Threshold - online_info = annotate_online( - net, [x["nick"] for x in results_parsed if x["src"] == "irc"] - ) - # Annotate the number of users in the channel - num_users = annotate_num_users( - net, [x["channel"] for x in results_parsed if x["src"] == "irc"] - ) - # Annotate the number channels the user is on - num_chans = annotate_num_chans( - net, [x["nick"] for x in results_parsed if x["src"] == "irc"] - ) - for item in results_parsed: - if item["nick"] in online_info: - item["online"] = online_info[item["nick"]] - if item["channel"] in num_users: - item["num_users"] = num_users[item["channel"]] - if item["nick"] in num_chans: - item["num_chans"] = num_chans[item["nick"]] - - context = { - "query": query, - "results": results_parsed, - "card": results["hits"]["total"]["value"], - "took": results["took"], - "redacted": results["redacted"], - "exemption": results["exemption"], - "fields": settings.OPENSEARCH_MAIN_SEARCH_FIELDS, - "sizes": settings.OPENSEARCH_MAIN_SIZES, - "timescales": settings.OPENSEARCH_MAIN_TIMESCALES, - } - return context - - -class Search(LoginRequiredMixin, View): - # parser_classes = [JSONParser] - template_name = "ui/results.html" - plan_name = "drilldown" - - def post(self, request): - if not request.user.has_plan(self.plan_name): - return HttpResponseForbidden() - - context = query_results(request, request.POST) - if not context: - return HttpResponseForbidden() - context["data"] = json.dumps( - [ - { - "text": item.get("msg", None) or item.get("id"), - "nick": item.get("nick", None), - "value": item.get("sentiment", None) or None, - "date": item.get("ts"), - } - for item in context["results"] - ] - ) - if context: - return render(request, self.template_name, context) - else: - return HttpResponse("No results") - - -class APISearch(LoginRequiredMixin, View): - # parser_classes = [JSONParser] - template_name = "ui/results.html" - plan_name = "drilldown" - - def post(self, request): - if not request.user.has_plan(self.plan_name): - return JsonResponse({"success": False}) - - context = query_results(request, request.POST) - return JsonResponse(context)