neptune/core/views/ui/drilldown.py

import json
import urllib
from copy import deepcopy

from django.conf import settings
from django.http import HttpResponse, JsonResponse
from django.shortcuts import render
from django.urls import reverse
from django.views import View
from django_tables2 import SingleTableView
from rest_framework.parsers import FormParser
from rest_framework.views import APIView

from core.lib.context import construct_query
from core.lib.opensearch import query_results
from core.lib.threshold import (
    annotate_num_chans,
    annotate_num_users,
    get_chans,
    get_users,
)
from core.views.helpers import hash_list, hash_lookup, randomise_list
from core.views.ui.tables import DrilldownTable


def parse_dates(dates):
    spl = dates.split(" - ")
    if all(spl):
        spl = [f"{x.replace(' ', 'T')}" for x in spl]
        if not len(spl) == 2:
            message = "Invalid dates"
            message_class = "danger"
            return {"message": message, "class": message_class}
        from_ts, to_ts = spl
        from_date, from_time = from_ts.split("T")
        to_date, to_time = to_ts.split("T")

        return {
            "from_date": from_date,
            "to_date": to_date,
            "from_time": from_time,
            "to_time": to_time,
        }


def create_tags(query):
    """
    Grab the tags out of the query and make a list
    we can add to the Bulma tags element when the page loads.
    """
    spl = query.split("AND")
    spl = [x.strip() for x in spl if ":" in x]
    spl = [x.replace('"', "") for x in spl]
    tags = [f"{tag}: {elem}" for tag, elem in [x.split(":")[:2] for x in spl]]
    return tags


def parse_tags(tags_pre):
    """
    Parse the tags from the variable tags_pre.
    """
    tags = {}
    tags_spl = tags_pre.split(",")
    if tags_spl:
        for tag in tags_spl:
            tag = tag.split(": ")
            if len(tag) == 2:
                key, val = tag
                tags[key] = val
    return tags


def make_table(context):
    table = DrilldownTable(context["object_list"])
    context["table"] = table
    # del context["results"]
    return context


def make_graph(results):
    graph = json.dumps(
        [
            {
                "text": item.get("msg", None) or item.get("id"),
                "nick": item.get("nick", None),
                "value": item.get("sentiment", None) or None,
                "date": item.get("ts"),
            }
            for item in results
        ]
    )
    return graph


def drilldown_search(request, return_context=False, template=None):
    extra_params = {}

    if not template:
        template_name = "ui/drilldown/table_results.html"
    else:
        template_name = template
    if request.user.is_anonymous:
        sizes = settings.OPENSEARCH_MAIN_SIZES_ANON
    else:
        sizes = settings.OPENSEARCH_MAIN_SIZES

    if request.GET:
        if not request.htmx:
            template_name = "ui/drilldown/drilldown.html"
        query_params = request.GET.dict()
    elif request.POST:
        query_params = request.POST.dict()
    else:
        template_name = "ui/drilldown/drilldown.html"
        context = {"sizes": sizes}
        return render(request, template_name, context)

    tmp_post = request.POST.dict()
    tmp_get = request.GET.dict()
    tmp_post = {k: v for k, v in tmp_post.items() if v and not v == "None"}
    tmp_get = {k: v for k, v in tmp_get.items() if v and not v == "None"}
    query_params.update(tmp_post)
    query_params.update(tmp_get)

    # URI we're passing to the template for linking
    if "csrfmiddlewaretoken" in query_params:
        del query_params["csrfmiddlewaretoken"]

    # Parse the dates
    if "dates" in query_params:
        dates = parse_dates(query_params["dates"])
        del query_params["dates"]
        if dates:
            if "message" in dates:
                return render(request, template_name, dates)
            query_params["from_date"] = dates["from_date"]
            query_params["to_date"] = dates["to_date"]
            query_params["from_time"] = dates["from_time"]
            query_params["to_time"] = dates["to_time"]

    if "query" in query_params:
        # Remove null values
        if query_params["query"] == "":
            del query_params["query"]
            # Turn the query into tags for populating the taglist
            # tags = create_tags(query_params["query"])
            # context["tags"] = tags
    # else:
    #     context = {"object_list": []}

    # Remove null values
    if "query_full" in query_params:
        if query_params["query_full"] == "":
            del query_params["query_full"]

    if "tags" in query_params:
        if query_params["tags"] == "":
            del query_params["tags"]
        else:
            tags = parse_tags(query_params["tags"])
            extra_params["tags"] = tags

    context = query_results(request, query_params, **extra_params)

    # Valid sizes
    context["sizes"] = sizes

    url_params = urllib.parse.urlencode(query_params)
    context["client_uri"] = url_params
    context["params"] = query_params
    if "message" in context:
        response = render(request, template_name, context)
        if request.GET:
            if request.htmx:
                response["HX-Push"] = reverse("home") + "?" + url_params
        elif request.POST:
            response["HX-Push"] = reverse("home") + "?" + url_params
        return response

    # Create data for chart.js sentiment graph
    graph = make_graph(context["object_list"])
    context["data"] = graph

    context = make_table(context)

    # URI we're passing to the template for linking, table fields removed
    table_fields = ["page", "sort"]
    clean_params = {k: v for k, v in query_params.items() if k not in table_fields}
    clean_url_params = urllib.parse.urlencode(clean_params)
    context["uri"] = clean_url_params

    # Warn users trying to use query string that the simple query supersedes it
    if all([x in query_params for x in ["query", "query_full"]]):
        context["message"] = (
            "You are searching with both query types. "
            "The simple query will be used. "
            "The full query will be ignored. "
            "Remove the text from the simple query if you wish "
            "to use the full query."
        )
        context["class"] = "warning"

    response = render(request, template_name, context)
    if request.GET:
        if request.htmx:
            response["HX-Push"] = reverse("home") + "?" + url_params
    elif request.POST:
        response["HX-Push"] = reverse("home") + "?" + url_params
    if return_context:
        return context
    return response


class DrilldownTableView(SingleTableView):
    table_class = DrilldownTable
    template_name = "ui/drilldown/table_results.html"
    paginate_by = settings.DRILLDOWN_RESULTS_PER_PAGE

    def get_queryset(self, request, **kwargs):
        context = drilldown_search(request, return_context=True)
        # Save the context as we will need to merge other attributes later
        self.context = context

        if "object_list" in context:
            return context["object_list"]
        else:
            return []

    def get(self, request, *args, **kwargs):
        self.object_list = self.get_queryset(request)
        show = []
        show = set().union(*(d.keys() for d in self.object_list))
        allow_empty = self.get_allow_empty()

        if not allow_empty:
            # When pagination is enabled and object_list is a queryset,
            # it's better to do a cheap query than to load the unpaginated
            # queryset in memory.
            if self.get_paginate_by(self.object_list) is not None and hasattr(
                self.object_list, "exists"
            ):
                is_empty = not self.object_list.exists()  # noqa
            else:
                is_empty = not self.object_list  # noqa
        context = self.get_context_data()
        if isinstance(self.context, HttpResponse):
            return self.context

        for k, v in self.context.items():
            if k not in context:
                context[k] = v
        context["show"] = show
        if request.method == "GET":
            if not request.htmx:
                self.template_name = "ui/drilldown/drilldown.html"
        response = self.render_to_response(context)
        # if not request.method == "GET":
        if "client_uri" in context:
            response["HX-Push"] = reverse("home") + "?" + context["client_uri"]
        return response

    def post(self, request, *args, **kwargs):
        return self.get(request, *args, **kwargs)


class Drilldown(View):
    template_name = "ui/drilldown/drilldown.html"
    plan_name = "drilldown"

    def get(self, request):
        return drilldown_search(request)

    def post(self, request):
        return drilldown_search(request)


class DrilldownContextModal(APIView):
    parser_classes = [FormParser]
    plan_name = "drilldown"
    template_name = "modals/context.html"

    def post(self, request):
        if request.resolver_match.url_name == "modal_context_table":
            self.template_name = "modals/context_table.html"

        size = 20
        nicks_sensitive = None
        query = False
        # Create the query params from the POST arguments
        mandatory = ["net", "channel", "num", "src", "index", "nick", "type", "mtype"]
        invalid = [None, False, "—", "None"]

        query_params = {k: v for k, v in request.data.items() if v}

        for key in query_params:
            if query_params[key] in invalid:
                query_params[key] = None
        for key in mandatory:
            if key not in query_params:
                query_params[key] = None

        # Lookup the hash values but don't disclose them to the user
        if settings.HASHING:
            SAFE_PARAMS = deepcopy(query_params)
            hash_lookup(request.user, SAFE_PARAMS)
        else:
            SAFE_PARAMS = query_params

        type = None
        if request.user.is_superuser:
            if "type" in query_params:
                type = query_params["type"]
                if type == "znc":
                    query_params["channel"] = "*status"
                    SAFE_PARAMS["channel"] = "*status"

            if type in ["query", "notice"]:
                nicks_sensitive = [
                    SAFE_PARAMS["channel"],
                    SAFE_PARAMS["nick"],
                ]  # UNSAFE
                # nicks = [query_params["channel"], query_params["nick"]]
                query = True

            if (
                query_params["index"] == "int"
                and query_params["mtype"] == "msg"
                and not type == "query"
            ):
                query_params["index"] = "main"
                SAFE_PARAMS["index"] = "main"

            if query_params["type"] in ["znc", "auth"]:
                query = True

        if not request.user.is_superuser:
            query_params["index"] = "main"
            SAFE_PARAMS["index"] = "main"

        query_params["sorting"] = "desc"
        SAFE_PARAMS["sorting"] = "desc"

        annotate = False
        if query_params["src"] == "irc":
            if query_params["type"] not in ["znc", "auth"]:
                annotate = True
        # Create the query with the context helper
        search_query = construct_query(
            query_params["index"],
            SAFE_PARAMS["net"],
            SAFE_PARAMS["channel"],
            query_params["src"],
            SAFE_PARAMS["num"],
            size,
            type=type,
            nicks=nicks_sensitive,
        )

        results = query_results(
            request,
            SAFE_PARAMS,
            annotate=annotate,
            custom_query=search_query,
            reverse=True,
            dedup_fields=["net", "type", "msg"],
            lookup_hashes=False,
        )
        if "message" in results:
            return render(request, self.template_name, results)

        if settings.HASHING:  # we probably want to see the tokens
            if not request.user.has_perm("bypass_hashing"):
                for index, item in enumerate(results["object_list"]):
                    if "tokens" in item:
                        results["object_list"][index]["msg"] = results["object_list"][
                            index
                        ].pop("tokens")
                        # item["msg"] = item.pop("tokens")

        # Make the time nicer
        # for index, item in enumerate(results["object_list"]):
        #    results["object_list"][index]["time"] = item["time"]+"SSS"
        context = {
            "net": query_params["net"],
            "channel": query_params["channel"],
            "src": query_params["src"],
            "ts": f"{query_params['date']} {query_params['time']}",
            "object_list": results["object_list"],
            "time": query_params["time"],
            "date": query_params["date"],
            "index": query_params["index"],
            "num": query_params["num"],
            "type": query_params["type"],
            "mtype": query_params["mtype"],
            "nick": query_params["nick"],
            "params": query_params,
        }
        if request.user.is_superuser:
            if query:
                context["query"] = True

        return render(request, self.template_name, context)


class ThresholdInfoModal(APIView):
    parser_classes = [FormParser]
    plan_name = "drilldown"
    template_name = "modals/drilldown.html"

    def post(self, request):
        # if not request.user.has_plan(self.plan_name):
        #    return JsonResponse({"success": False})
        if "net" not in request.data:
            return JsonResponse({"success": False})
        if "nick" not in request.data:
            return JsonResponse({"success": False})
        if "channel" not in request.data:
            return JsonResponse({"success": False})

        net = request.data["net"]
        nick = request.data["nick"]
        channel = request.data["channel"]

        # SAFE BLOCK #
        # Lookup the hash values but don't disclose them to the user
        if settings.HASHING:
            SAFE_PARAMS = request.data.dict()
            hash_lookup(request.user, SAFE_PARAMS)
        safe_net = SAFE_PARAMS["net"]
        safe_nick = SAFE_PARAMS["nick"]
        safe_channel = SAFE_PARAMS["channel"]
        channels = get_chans(safe_net, [safe_nick])
        users = get_users(safe_net, [safe_channel])
        num_users = annotate_num_users(safe_net, channels)
        num_chans = annotate_num_chans(safe_net, users)
        if channels:
            inter_users = get_users(safe_net, channels)
        else:
            inter_users = []
        if users:
            inter_chans = get_chans(safe_net, users)
        else:
            inter_chans = []
        if settings.HASHING:
            hash_list(request.user, inter_chans)
            hash_list(request.user, inter_users)

            hash_list(request.user, num_chans, hash_keys=True)
            hash_list(request.user, num_users, hash_keys=True)

            hash_list(request.user, channels)
            hash_list(request.user, users)

        if settings.RANDOMISATION:
            randomise_list(request.user, num_chans)
            randomise_list(request.user, num_users)

        # SAFE BLOCK END #

        context = {
            "net": net,
            "nick": nick,
            "channel": channel,
            "chans": channels,
            "users": users,
            "inter_chans": inter_chans,
            "inter_users": inter_users,
            "num_users": num_users,
            "num_chans": num_chans,
        }
        return render(request, self.template_name, context)