import json import urllib from copy import deepcopy from django.conf import settings from django.http import HttpResponse, JsonResponse from django.shortcuts import render from django.urls import reverse from django.views import View from django_tables2 import SingleTableView from rest_framework.parsers import FormParser from rest_framework.views import APIView from core.lib.context import construct_query from core.lib.opensearch import query_results from core.lib.threshold import ( annotate_num_chans, annotate_num_users, get_chans, get_users, ) from core.views.helpers import hash_list, hash_lookup, randomise_list from core.views.ui.tables import DrilldownTable def parse_dates(dates): spl = dates.split(" - ") if all(spl): spl = [f"{x.replace(' ', 'T')}" for x in spl] if not len(spl) == 2: message = "Invalid dates" message_class = "danger" return {"message": message, "class": message_class} from_ts, to_ts = spl from_date, from_time = from_ts.split("T") to_date, to_time = to_ts.split("T") return { "from_date": from_date, "to_date": to_date, "from_time": from_time, "to_time": to_time, } def create_tags(query): """ Grab the tags out of the query and make a list we can add to the Bulma tags element when the page loads. """ spl = query.split("AND") spl = [x.strip() for x in spl if ":" in x] spl = [x.replace('"', "") for x in spl] tags = [f"{tag}: {elem}" for tag, elem in [x.split(":")[:2] for x in spl]] return tags def parse_tags(tags_pre): """ Parse the tags from the variable tags_pre. """ tags = {} tags_spl = tags_pre.split(",") if tags_spl: for tag in tags_spl: tag = tag.split(": ") if len(tag) == 2: key, val = tag tags[key] = val return tags def make_table(context): table = DrilldownTable(context["object_list"]) context["table"] = table # del context["results"] return context def make_graph(results): graph = [] for index, item in enumerate(results): date = str(index) graph.append( { "text": item.get("tokens", None) or item.get("msg", None) or item.get("id"), "nick": item.get("nick", None), "channel": item.get("channel", None), "net": item.get("net", None), "value": item.get("sentiment", None) or None, "date": date, } ) return json.dumps(graph) def drilldown_search(request, return_context=False, template=None): extra_params = {} if not template: template_name = "ui/drilldown/table_results.html" else: template_name = template if request.user.is_anonymous: sizes = settings.OPENSEARCH_MAIN_SIZES_ANON else: sizes = settings.OPENSEARCH_MAIN_SIZES if request.GET: if not request.htmx: template_name = "ui/drilldown/drilldown.html" query_params = request.GET.dict() elif request.POST: query_params = request.POST.dict() else: template_name = "ui/drilldown/drilldown.html" context = {"sizes": sizes} return render(request, template_name, context) tmp_post = request.POST.dict() tmp_get = request.GET.dict() tmp_post = {k: v for k, v in tmp_post.items() if v and not v == "None"} tmp_get = {k: v for k, v in tmp_get.items() if v and not v == "None"} query_params.update(tmp_post) query_params.update(tmp_get) # URI we're passing to the template for linking if "csrfmiddlewaretoken" in query_params: del query_params["csrfmiddlewaretoken"] # Parse the dates if "dates" in query_params: dates = parse_dates(query_params["dates"]) del query_params["dates"] if dates: if "message" in dates: return render(request, template_name, dates) query_params["from_date"] = dates["from_date"] query_params["to_date"] = dates["to_date"] query_params["from_time"] = dates["from_time"] query_params["to_time"] = dates["to_time"] if "query" in query_params: # Remove null values if query_params["query"] == "": del query_params["query"] # Turn the query into tags for populating the taglist # tags = create_tags(query_params["query"]) # context["tags"] = tags # else: # context = {"object_list": []} # Remove null values if "query_full" in query_params: if query_params["query_full"] == "": del query_params["query_full"] if "tags" in query_params: if query_params["tags"] == "": del query_params["tags"] else: tags = parse_tags(query_params["tags"]) extra_params["tags"] = tags context = query_results(request, query_params, **extra_params) # Valid sizes context["sizes"] = sizes url_params = urllib.parse.urlencode(query_params) context["client_uri"] = url_params context["params"] = query_params if "message" in context: response = render(request, template_name, context) if request.GET: if request.htmx: response["HX-Push"] = reverse("home") + "?" + url_params elif request.POST: response["HX-Push"] = reverse("home") + "?" + url_params return response # Create data for chart.js sentiment graph graph = make_graph(context["object_list"]) context["data"] = graph context = make_table(context) # URI we're passing to the template for linking, table fields removed table_fields = ["page", "sort"] clean_params = {k: v for k, v in query_params.items() if k not in table_fields} clean_url_params = urllib.parse.urlencode(clean_params) context["uri"] = clean_url_params # Warn users trying to use query string that the simple query supersedes it if all([x in query_params for x in ["query", "query_full"]]): context["message"] = ( "You are searching with both query types. " "The simple query will be used. " "The full query will be ignored. " "Remove the text from the simple query if you wish " "to use the full query." ) context["class"] = "warning" response = render(request, template_name, context) if request.GET: if request.htmx: response["HX-Push"] = reverse("home") + "?" + url_params elif request.POST: response["HX-Push"] = reverse("home") + "?" + url_params if return_context: return context return response class DrilldownTableView(SingleTableView): table_class = DrilldownTable template_name = "ui/drilldown/table_results.html" paginate_by = settings.DRILLDOWN_RESULTS_PER_PAGE def get_queryset(self, request, **kwargs): context = drilldown_search(request, return_context=True) # Save the context as we will need to merge other attributes later self.context = context if "object_list" in context: return context["object_list"] else: return [] def get(self, request, *args, **kwargs): self.object_list = self.get_queryset(request) show = [] show = set().union(*(d.keys() for d in self.object_list)) allow_empty = self.get_allow_empty() if not allow_empty: # When pagination is enabled and object_list is a queryset, # it's better to do a cheap query than to load the unpaginated # queryset in memory. if self.get_paginate_by(self.object_list) is not None and hasattr( self.object_list, "exists" ): is_empty = not self.object_list.exists() # noqa else: is_empty = not self.object_list # noqa context = self.get_context_data() if isinstance(self.context, HttpResponse): return self.context for k, v in self.context.items(): if k not in context: context[k] = v context["show"] = show if request.method == "GET": if not request.htmx: self.template_name = "ui/drilldown/drilldown.html" response = self.render_to_response(context) # if not request.method == "GET": if "client_uri" in context: response["HX-Push"] = reverse("home") + "?" + context["client_uri"] return response def post(self, request, *args, **kwargs): return self.get(request, *args, **kwargs) class Drilldown(View): template_name = "ui/drilldown/drilldown.html" plan_name = "drilldown" def get(self, request): return drilldown_search(request) def post(self, request): return drilldown_search(request) class DrilldownContextModal(APIView): parser_classes = [FormParser] plan_name = "drilldown" template_name = "modals/context.html" def post(self, request): if request.resolver_match.url_name == "modal_context_table": self.template_name = "modals/context_table.html" size = 20 nicks_sensitive = None query = False # Create the query params from the POST arguments mandatory = ["net", "channel", "num", "src", "index", "nick", "type", "mtype"] invalid = [None, False, "—", "None"] query_params = {k: v for k, v in request.data.items() if v} for key in query_params: if query_params[key] in invalid: query_params[key] = None for key in mandatory: if key not in query_params: query_params[key] = None # Lookup the hash values but don't disclose them to the user if settings.HASHING: SAFE_PARAMS = deepcopy(query_params) hash_lookup(request.user, SAFE_PARAMS) else: SAFE_PARAMS = query_params type = None if request.user.is_superuser: if "type" in query_params: type = query_params["type"] if type == "znc": query_params["channel"] = "*status" SAFE_PARAMS["channel"] = "*status" if type in ["query", "notice"]: nicks_sensitive = [ SAFE_PARAMS["channel"], SAFE_PARAMS["nick"], ] # UNSAFE # nicks = [query_params["channel"], query_params["nick"]] query = True if ( query_params["index"] == "int" and query_params["mtype"] == "msg" and not type == "query" ): query_params["index"] = "main" SAFE_PARAMS["index"] = "main" if query_params["type"] in ["znc", "auth"]: query = True if not request.user.is_superuser: query_params["index"] = "main" SAFE_PARAMS["index"] = "main" query_params["sorting"] = "desc" SAFE_PARAMS["sorting"] = "desc" annotate = False if query_params["src"] == "irc": if query_params["type"] not in ["znc", "auth"]: annotate = True # Create the query with the context helper search_query = construct_query( query_params["index"], SAFE_PARAMS["net"], SAFE_PARAMS["channel"], query_params["src"], SAFE_PARAMS["num"], size, type=type, nicks=nicks_sensitive, ) results = query_results( request, SAFE_PARAMS, annotate=annotate, custom_query=search_query, reverse=True, dedup_fields=["net", "type", "msg"], lookup_hashes=False, ) if "message" in results: return render(request, self.template_name, results) if settings.HASHING: # we probably want to see the tokens if not request.user.has_perm("core.bypass_hashing"): for index, item in enumerate(results["object_list"]): if "tokens" in item: results["object_list"][index]["msg"] = results["object_list"][ index ].pop("tokens") # item["msg"] = item.pop("tokens") # Make the time nicer # for index, item in enumerate(results["object_list"]): # results["object_list"][index]["time"] = item["time"]+"SSS" context = { "net": query_params["net"], "channel": query_params["channel"], "src": query_params["src"], "ts": f"{query_params['date']} {query_params['time']}", "object_list": results["object_list"], "time": query_params["time"], "date": query_params["date"], "index": query_params["index"], "num": query_params["num"], "type": query_params["type"], "mtype": query_params["mtype"], "nick": query_params["nick"], "params": query_params, } if request.user.is_superuser: if query: context["query"] = True return render(request, self.template_name, context) class ThresholdInfoModal(APIView): parser_classes = [FormParser] plan_name = "drilldown" template_name = "modals/drilldown.html" def post(self, request, type=None): # if not request.user.has_plan(self.plan_name): # return JsonResponse({"success": False}) if "net" not in request.data: return JsonResponse({"success": False}) if "nick" not in request.data: return JsonResponse({"success": False}) if "channel" not in request.data: return JsonResponse({"success": False}) if type == "window": self.template_name = "windows/drilldown.html" net = request.data["net"] nick = request.data["nick"] channel = request.data["channel"] # SAFE BLOCK # # Lookup the hash values but don't disclose them to the user if settings.HASHING: SAFE_PARAMS = request.data.dict() hash_lookup(request.user, SAFE_PARAMS) safe_net = SAFE_PARAMS["net"] safe_nick = SAFE_PARAMS["nick"] safe_channel = SAFE_PARAMS["channel"] channels = get_chans(safe_net, [safe_nick]) users = get_users(safe_net, [safe_channel]) num_users = annotate_num_users(safe_net, channels) num_chans = annotate_num_chans(safe_net, users) if channels: inter_users = get_users(safe_net, channels) else: inter_users = [] if users: inter_chans = get_chans(safe_net, users) else: inter_chans = [] if settings.HASHING: hash_list(request.user, inter_chans) hash_list(request.user, inter_users) hash_list(request.user, num_chans, hash_keys=True) hash_list(request.user, num_users, hash_keys=True) hash_list(request.user, channels) hash_list(request.user, users) if settings.RANDOMISATION: randomise_list(request.user, num_chans) randomise_list(request.user, num_users) # SAFE BLOCK END # context = { "net": net, "nick": nick, "channel": channel, "chans": channels, "users": users, "inter_chans": inter_chans, "inter_users": inter_users, "num_users": num_users, "num_chans": num_chans, } return render(request, self.template_name, context)