neptune/core/views/ui/drilldown.py

import json
import urllib
import uuid

from django.conf import settings
from django.http import HttpResponse, JsonResponse
from django.shortcuts import render
from django.urls import reverse
from django.views import View
from django_tables2 import SingleTableView
from rest_framework.parsers import FormParser
from rest_framework.views import APIView

from core.lib.context import construct_query

# from core.lib.opensearch import query_results
from core.lib.manticore import query_results
from core.lib.threshold import (
    annotate_num_chans,
    annotate_num_users,
    get_chans,
    get_users,
)
from core.views.ui.tables import DrilldownTable

# from copy import deepcopy


def parse_dates(dates):
    spl = dates.split(" - ")
    if all(spl):
        spl = [f"{x.replace(' ', 'T')}" for x in spl]
        if not len(spl) == 2:
            message = "Invalid dates"
            message_class = "danger"
            return {"message": message, "class": message_class}
        from_ts, to_ts = spl
        from_date, from_time = from_ts.split("T")
        to_date, to_time = to_ts.split("T")

        return {
            "from_date": from_date,
            "to_date": to_date,
            "from_time": from_time,
            "to_time": to_time,
        }


def create_tags(query):
    """
    Grab the tags out of the query and make a list
    we can add to the Bulma tags element when the page loads.
    """
    spl = query.split("AND")
    spl = [x.strip() for x in spl if ":" in x]
    spl = [x.replace('"', "") for x in spl]
    tags = [f"{tag}: {elem}" for tag, elem in [x.split(":")[:2] for x in spl]]
    return tags


def parse_tags(tags_pre):
    """
    Parse the tags from the variable tags_pre.
    """
    tags = {}
    tags_spl = tags_pre.split(",")
    if tags_spl:
        for tag in tags_spl:
            tag = tag.split(": ")
            if len(tag) == 2:
                key, val = tag
                tags[key] = val
    return tags


def make_table(context):
    table = DrilldownTable(context["object_list"])
    context["table"] = table
    # del context["results"]
    return context


def make_graph(results):
    graph = []
    for index, item in enumerate(results):
        date = str(index)
        graph.append(
            {
                "text": item.get("tokens", None)
                or item.get("msg", None)
                or item.get("id"),
                "nick": item.get("nick", None),
                "channel": item.get("channel", None),
                "net": item.get("net", None),
                "value": item.get("sentiment", None) or None,
                "date": date,
            }
        )
    return json.dumps(graph)


def drilldown_search(request, return_context=False, template=None):
    extra_params = {}

    if not template:
        template_name = "widgets/table_results.html"
    else:
        template_name = template
    if request.user.is_anonymous:
        sizes = settings.OPENSEARCH_MAIN_SIZES_ANON
    else:
        sizes = settings.OPENSEARCH_MAIN_SIZES

    if request.GET:
        if not request.htmx:
            template_name = "ui/drilldown/drilldown.html"
        query_params = request.GET.dict()
    elif request.POST:
        query_params = request.POST.dict()
    else:
        template_name = "ui/drilldown/drilldown.html"
        context = {"sizes": sizes, "unique": "results"}
        return render(request, template_name, context)

    tmp_post = request.POST.dict()
    tmp_get = request.GET.dict()
    tmp_post = {k: v for k, v in tmp_post.items() if v and not v == "None"}
    tmp_get = {k: v for k, v in tmp_get.items() if v and not v == "None"}
    query_params.update(tmp_post)
    query_params.update(tmp_get)

    # URI we're passing to the template for linking
    if "csrfmiddlewaretoken" in query_params:
        del query_params["csrfmiddlewaretoken"]

    # Parse the dates
    if "dates" in query_params:
        dates = parse_dates(query_params["dates"])
        del query_params["dates"]
        if dates:
            if "message" in dates:
                return render(request, template_name, dates)
            query_params["from_date"] = dates["from_date"]
            query_params["to_date"] = dates["to_date"]
            query_params["from_time"] = dates["from_time"]
            query_params["to_time"] = dates["to_time"]

    if "query" in query_params:
        # Remove null values
        if query_params["query"] == "":
            del query_params["query"]
            # Turn the query into tags for populating the taglist
            # tags = create_tags(query_params["query"])
            # context["tags"] = tags
    # else:
    #     context = {"object_list": []}

    # Remove null values
    if "query_full" in query_params:
        if query_params["query_full"] == "":
            del query_params["query_full"]

    if "tags" in query_params:
        if query_params["tags"] == "":
            del query_params["tags"]
        else:
            tags = parse_tags(query_params["tags"])
            extra_params["tags"] = tags

    context = query_results(request, query_params, **extra_params)
    context["unique"] = "results"

    # Valid sizes
    context["sizes"] = sizes

    url_params = urllib.parse.urlencode(query_params)
    context["client_uri"] = url_params
    context["params"] = query_params
    if "message" in context:
        if return_context:
            return context
        response = render(request, template_name, context)
        if request.GET:
            if request.htmx:
                response["HX-Push"] = reverse("home") + "?" + url_params
        elif request.POST:
            response["HX-Push"] = reverse("home") + "?" + url_params
        return response

    # Create data for chart.js sentiment graph
    graph = make_graph(context["object_list"])
    context["data"] = graph

    context = make_table(context)

    # URI we're passing to the template for linking, table fields removed
    table_fields = ["page", "sort"]
    clean_params = {k: v for k, v in query_params.items() if k not in table_fields}
    clean_url_params = urllib.parse.urlencode(clean_params)
    context["uri"] = clean_url_params

    # Warn users trying to use query string that the simple query supersedes it
    if all([x in query_params for x in ["query", "query_full"]]):
        context["message"] = (
            "You are searching with both query types. "
            "The simple query will be used. "
            "The full query will be ignored. "
            "Remove the text from the simple query if you wish "
            "to use the full query."
        )
        context["class"] = "warning"

    # unique = str(uuid.uuid4())[:8]
    response = render(request, template_name, context)
    if request.GET:
        if request.htmx:
            response["HX-Push"] = reverse("home") + "?" + url_params
    elif request.POST:
        response["HX-Push"] = reverse("home") + "?" + url_params
    if return_context:
        return context
    return response


class DrilldownTableView(SingleTableView):
    table_class = DrilldownTable
    template_name = "widgets/table_results.html"
    paginate_by = settings.DRILLDOWN_RESULTS_PER_PAGE

    def get_queryset(self, request, **kwargs):
        context = drilldown_search(request, return_context=True)
        # Save the context as we will need to merge other attributes later
        self.context = context

        if "object_list" in context:
            return context["object_list"]
        else:
            return []

    def get(self, request, *args, **kwargs):
        self.object_list = self.get_queryset(request)
        show = []
        show = set().union(*(d.keys() for d in self.object_list))
        allow_empty = self.get_allow_empty()

        if not allow_empty:
            # When pagination is enabled and object_list is a queryset,
            # it's better to do a cheap query than to load the unpaginated
            # queryset in memory.
            if self.get_paginate_by(self.object_list) is not None and hasattr(
                self.object_list, "exists"
            ):
                is_empty = not self.object_list.exists()  # noqa
            else:
                is_empty = not self.object_list  # noqa
        context = self.get_context_data()
        if isinstance(self.context, HttpResponse):
            return self.context

        for k, v in self.context.items():
            if k not in context:
                context[k] = v
        context["show"] = show

        if request.method == "GET":
            if not request.htmx:
                self.template_name = "ui/drilldown/drilldown.html"
        response = self.render_to_response(context)
        # if not request.method == "GET":
        if "client_uri" in context:
            response["HX-Push"] = reverse("home") + "?" + context["client_uri"]
        return response

    def post(self, request, *args, **kwargs):
        return self.get(request, *args, **kwargs)


class Drilldown(View):
    template_name = "ui/drilldown/drilldown.html"
    plan_name = "drilldown"

    def get(self, request):
        return drilldown_search(request)

    def post(self, request):
        return drilldown_search(request)


class DrilldownContextModal(APIView):
    parser_classes = [FormParser]
    plan_name = "drilldown"
    template_name = "modals/context.html"

    def post(self, request):
        if request.resolver_match.url_name == "modal_context_table":
            self.template_name = "partials/context_table.html"

        size = 20
        nicks_sensitive = None
        query = False
        # Create the query params from the POST arguments
        mandatory = [
            "net",
            "channel",
            "num",
            "source",
            "index",
            "nick",
            "type",
            "mtype",
        ]
        invalid = [None, False, "—", "None"]

        query_params = {k: v for k, v in request.data.items() if v}
        for key in query_params:
            if query_params[key] in invalid:
                query_params[key] = None
        for key in mandatory:
            if key not in query_params:
                query_params[key] = None

        # Lookup the hash values but don't disclose them to the user
        # if settings.HASHING:
        #     if query_params["source"] not in settings.SAFE_SOURCES:
        #         SAFE_PARAMS = deepcopy(query_params)
        #         hash_lookup(request.user, SAFE_PARAMS)
        #     else:
        #         SAFE_PARAMS = deepcopy(query_params)
        # else:
        #     SAFE_PARAMS = query_params

        type = None
        if request.user.is_superuser:
            if "type" in query_params:
                type = query_params["type"]
                if type == "znc":
                    query_params["channel"] = "*status"
                    # SAFE_PARAMS["channel"] = "*status"

            if type in ["query", "notice"]:
                nicks_sensitive = [
                    query_params["channel"],
                    query_params["nick"],
                ]  # UNSAFE
                # nicks = [query_params["channel"], query_params["nick"]]
                query = True

            if (
                query_params["index"] == "int"
                and query_params["mtype"] == "msg"
                and not type == "query"
            ):
                query_params["index"] = "main"
                # SAFE_PARAMS["index"] = "main"

            if query_params["type"] in ["znc", "auth"]:
                query = True

        if not request.user.is_superuser:
            query_params["index"] = "main"
            # SAFE_PARAMS["index"] = "main"

        query_params["sorting"] = "desc"
        # SAFE_PARAMS["sorting"] = "desc"

        annotate = False
        if query_params["source"] == "irc":
            if query_params["type"] not in ["znc", "auth"]:
                annotate = True
        # Create the query with the context helper
        if query_params["num"].isdigit():
            query_params["num"] = int(query_params["num"])
        search_query = construct_query(
            query_params["index"],
            query_params["net"],
            query_params["channel"],
            query_params["source"],
            query_params["num"],
            size,
            type=type,
            nicks=nicks_sensitive,
        )

        results = query_results(
            request,
            query_params,
            annotate=annotate,
            custom_query=search_query,
            reverse=True,
            dedup_fields=["net", "type", "msg"],
        )
        if "message" in results:
            return render(request, self.template_name, results)

        # if settings.HASHING:  # we probably want to see the tokens
        #     if query_params["source"] not in settings.SAFE_SOURCES:
        #         if not request.user.has_perm("core.bypass_hashing"):
        #             for index, item in enumerate(results["object_list"]):
        #                 if "tokens" in item:
        #                     results["object_list"][index]["msg"] = results[
        #                         "object_list"
        #                     ][index].pop("tokens")
        #                     # item["msg"] = item.pop("tokens")

        # Make the time nicer
        # for index, item in enumerate(results["object_list"]):
        #    results["object_list"][index]["time"] = item["time"]+"SSS"
        unique = str(uuid.uuid4())[:8]
        print("PARAMS", query_params)
        context = {
            "net": query_params["net"],
            "channel": query_params["channel"],
            "source": query_params["source"],
            "ts": f"{query_params['date']} {query_params['time']}",
            "object_list": results["object_list"],
            "time": query_params["time"],
            "date": query_params["date"],
            "index": query_params["index"],
            "num": query_params["num"],
            "type": query_params["type"],
            "mtype": query_params["mtype"],
            "nick": query_params["nick"],
            "params": query_params,
            "unique": unique,
        }
        if request.user.is_superuser:
            if query:
                context["query"] = True

        return render(request, self.template_name, context)


class ThresholdInfoModal(APIView):
    parser_classes = [FormParser]
    plan_name = "drilldown"
    template_name = "modals/drilldown.html"

    def post(self, request, type=None):
        # if not request.user.has_plan(self.plan_name):
        #    return JsonResponse({"success": False})
        if "net" not in request.data:
            return JsonResponse({"success": False})
        if "nick" not in request.data:
            return JsonResponse({"success": False})
        if "channel" not in request.data:
            return JsonResponse({"success": False})
        if type == "window":
            self.template_name = "windows/drilldown.html"
        elif type == "widget":
            self.template_name = "widgets/drilldown.html"

        net = request.data["net"]
        nick = request.data["nick"]
        channel = request.data["channel"]

        # SAFE BLOCK #
        # Lookup the hash values but don't disclose them to the user
        # if settings.HASHING:
        #     SAFE_PARAMS = request.data.dict()
        #     hash_lookup(request.user, SAFE_PARAMS)

        channels = get_chans(net, [nick])
        users = get_users(net, [nick])
        num_users = annotate_num_users(net, channels)
        num_chans = annotate_num_chans(net, users)
        if channels:
            inter_users = get_users(net, channels)
        else:
            inter_users = []
        if users:
            inter_chans = get_chans(net, users)
        else:
            inter_chans = []
        # if settings.HASHING:
        #     hash_list(request.user, inter_chans)
        #     hash_list(request.user, inter_users)

        #     hash_list(request.user, num_chans, hash_keys=True)
        #     hash_list(request.user, num_users, hash_keys=True)

        #     hash_list(request.user, channels)
        #     hash_list(request.user, users)

        # if settings.RANDOMISATION:
        #     randomise_list(request.user, num_chans)
        #     randomise_list(request.user, num_users)

        # SAFE BLOCK END #

        unique = str(uuid.uuid4())[:8]
        context = {
            "net": net,
            "nick": nick,
            "channel": channel,
            "chans": channels,
            "users": users,
            "inter_chans": inter_chans,
            "inter_users": inter_users,
            "num_users": num_users,
            "num_chans": num_chans,
            "unique": unique,
        }
        return render(request, self.template_name, context)