Implement Insights page

app/urls.py

@@ -19,16 +19,18 @@ from django.contrib import admin
 from django.urls import include, path
 from django.views.generic import TemplateView
 
-from core.api.views.threshold import (
+# Threshold API stuff
-    ThresholdChans,
+from core.api.views.threshold import ThresholdChans, ThresholdOnline, ThresholdUsers
-    ThresholdInfoModal,
+
-    ThresholdOnline,
+# Main tool pages
-    ThresholdUsers,
-)
 from core.ui.views.drilldown import Drilldown
+from core.ui.views.insights import Insights
 from core.views import Billing, Cancel, Home, Order, Portal, Signup
 from core.views.callbacks import Callback
-from core.views.dynamic.search import APISearch, Search
+
+# Dynamic elements
+from core.views.dynamic.drilldown import DrilldownSearch, ThresholdInfoModal
+from core.views.dynamic.insights import InsightsInfoModal, InsightsSearch
 
 urlpatterns = [
     path("", Home.as_view(), name="home"),
@@ -49,10 +51,12 @@ urlpatterns = [
     path("accounts/", include("django.contrib.auth.urls")),
     path("accounts/signup/", Signup.as_view(), name="signup"),
     path("ui/drilldown/", Drilldown.as_view(), name="drilldown"),
-    path("parts/search/", Search.as_view(), name="search"),
+    path("ui/insights/", Insights.as_view(), name="insights"),
-    path("modal/info/", ThresholdInfoModal.as_view(), name="modal_info"),
+    path("parts/search/drilldown/", DrilldownSearch.as_view(), name="search_drilldown"),
+    path("parts/search/insights/", InsightsSearch.as_view(), name="search_insights"),
+    path("modal/drilldown/", ThresholdInfoModal.as_view(), name="modal_drilldown"),
+    path("modal/insights/", InsightsInfoModal.as_view(), name="modal_insights"),
     path("api/chans/", ThresholdChans.as_view(), name="chans"),
     path("api/users/", ThresholdUsers.as_view(), name="users"),
     path("api/online/", ThresholdOnline.as_view(), name="online"),
-    path("api/search/", APISearch.as_view(), name="api_search"),
 ] + static(settings.STATIC_URL, document_root=settings.STATIC_ROOT)

core/api/views/threshold.py

@@ -2,17 +2,10 @@ import logging
 
 from django.contrib.auth.mixins import LoginRequiredMixin
 from django.http import HttpResponse, JsonResponse
-from django.shortcuts import render
 from rest_framework.parsers import FormParser
 from rest_framework.views import APIView
 
-from core.lib.threshold import (
+from core.lib.threshold import annotate_online, get_chans, get_users
-    annotate_num_chans,
-    annotate_num_users,
-    annotate_online,
-    get_chans,
-    get_users,
-)
 
 logger = logging.getLogger(__name__)
 
@@ -72,46 +65,3 @@ class ThresholdOnline(LoginRequiredMixin, APIView):
         query = request.data["query"]
         online_info = annotate_online(net, query)
         return JsonResponse(online_info)
-
-
-class ThresholdInfoModal(LoginRequiredMixin, APIView):
-    parser_classes = [FormParser]
-    plan_name = "drilldown"
-    template_name = "modals/info.html"
-
-    def post(self, request):
-        if not request.user.has_plan(self.plan_name):
-            return JsonResponse({"success": False})
-        if "net" not in request.data:
-            return JsonResponse({"success": False})
-        if "nick" not in request.data:
-            return JsonResponse({"success": False})
-        if "channel" not in request.data:
-            return JsonResponse({"success": False})
-        net = request.data["net"]
-        nick = request.data["nick"]
-        channel = request.data["channel"]
-        channels = get_chans(net, [nick])
-        users = get_users(net, [channel])
-        num_users = annotate_num_users(net, channels)
-        num_chans = annotate_num_chans(net, users)
-        if channels:
-            inter_users = get_users(net, channels)
-        else:
-            inter_users = []
-        if users:
-            inter_chans = get_chans(net, users)
-        else:
-            inter_chans = []
-        context = {
-            "net": net,
-            "nick": nick,
-            "channel": channel,
-            "chans": channels,
-            "users": users,
-            "inter_chans": inter_chans,
-            "inter_users": inter_users,
-            "num_users": num_users,
-            "num_chans": num_chans,
-        }
-        return render(request, self.template_name, context)

core/lib/opensearch.py

@@ -2,8 +2,13 @@ from django.conf import settings
 from opensearchpy import OpenSearch
 from opensearchpy.exceptions import RequestError
 
+from core.lib.threshold import annotate_num_chans, annotate_num_users, annotate_online
+
 
 def initialise_opensearch():
+    """
+    Inititialise the OpenSearch API endpoint.
+    """
     auth = (settings.OPENSEARCH_USERNAME, settings.OPENSEARCH_PASSWORD)
     client = OpenSearch(
         # fmt: off
@@ -22,9 +27,157 @@ def initialise_opensearch():
     return client
 
 
-def construct_query(query, fields, size):
+client = initialise_opensearch()
-    if not fields:
+
-        fields = settings.OPENSEARCH_MAIN_SEARCH_FIELDS
+
+def annotate_results(results_parsed):
+    """
+    Accept a list of dict objects, search for the number of channels and users.
+    Add them to the object.
+    Mutate it in place. Does not return anything.
+    """
+    # Figure out items with net (not discord)
+    nets = set()
+    for x in results_parsed:
+        if "net" in x:
+            nets.add(x["net"])
+
+    for net in nets:
+        # Annotate the online attribute from Threshold
+        online_info = annotate_online(
+            net, [x["nick"] for x in results_parsed if x["src"] == "irc"]
+        )
+        # Annotate the number of users in the channel
+        num_users = annotate_num_users(
+            net, [x["channel"] for x in results_parsed if x["src"] == "irc"]
+        )
+        # Annotate the number channels the user is on
+        num_chans = annotate_num_chans(
+            net, [x["nick"] for x in results_parsed if x["src"] == "irc"]
+        )
+        for item in results_parsed:
+            if item["nick"] in online_info:
+                item["online"] = online_info[item["nick"]]
+            if item["channel"] in num_users:
+                item["num_users"] = num_users[item["channel"]]
+            if item["nick"] in num_chans:
+                item["num_chans"] = num_chans[item["nick"]]
+
+
+def filter_blacklisted(user, response):
+    """
+    Low level filter to take the raw OpenSearch response and remove
+    objects from it we want to keep secret.
+    Does not return, the object is mutated in place.
+    """
+    response["redacted"] = 0
+    response["exemption"] = None
+    # For every hit from ES
+    for item in list(response["hits"]["hits"]):
+        # For every blacklisted type
+        for blacklisted_type in settings.OPENSEARCH_BLACKLISTED.keys():
+            # Check this field we are matching exists
+            if blacklisted_type in item["_source"].keys():
+                content = item["_source"][blacklisted_type]
+                # For every item in the blacklisted array for the type
+                for blacklisted_item in settings.OPENSEARCH_BLACKLISTED[
+                    blacklisted_type
+                ]:
+                    if blacklisted_item in str(content):
+                        # Remove the item
+                        if item in response["hits"]["hits"]:
+                            if not user.is_superuser:
+                                response["hits"]["hits"].remove(item)
+                            # Let the UI know something was redacted
+                            response["redacted"] += 1
+                            response["exemption"] = True
+
+
+def run_main_query(client, user, query, size=None):
+    """
+    Low level helper to run an ES query.
+    Accept a user to pass it to the filter, so we can
+    avoid filtering for superusers.
+    Accept fields and size, for the fields we want to match and the
+    number of results to return.
+    """
+    search_query = construct_query(query, size)
+    try:
+        response = client.search(
+            body=search_query, index=settings.OPENSEARCH_INDEX_MAIN
+        )
+    except RequestError:
+        print("REQUEST ERROR")
+        return False
+    filter_blacklisted(user, response)
+    return response
+
+
+def query_results(request, size=None):
+    """
+    API helper to alter the OpenSearch return format into something
+    a bit better to parse.
+    Accept a HTTP request object. Run the query, and annotate the
+    results with the other data we have.
+    """
+    if not size:
+        if "size" in request.POST:
+            size = request.POST["size"]
+            if size not in settings.OPENSEARCH_MAIN_SIZES:
+                return False
+    if "query" in request.POST:
+        query = request.POST["query"]
+        results = run_main_query(
+            client,
+            request.user,
+            query,
+            size,
+        )
+        if not results:
+            return False
+        results_parsed = []
+        if "hits" in results.keys():
+            if "hits" in results["hits"]:
+                for item in results["hits"]["hits"]:
+                    element = item["_source"]
+                    element["id"] = item["_id"]
+
+                    # Split the timestamp into date and time
+                    ts = element["ts"]
+                    ts_spl = ts.split("T")
+                    date = ts_spl[0]
+                    time = ts_spl[1]
+                    element["date"] = date
+                    element["time"] = time
+                    results_parsed.append(element)
+
+        annotate_results(results_parsed)
+
+        context = {
+            "query": query,
+            "results": results_parsed,
+            "card": results["hits"]["total"]["value"],
+            "took": results["took"],
+            "redacted": results["redacted"],
+            "exemption": results["exemption"],
+        }
+        return context
+
+
+def query_single_result(request):
+    context = query_results(request, 1)
+    dedup_set = {item["nick"] for item in context["results"]}
+    if len(dedup_set) == 1:
+        context["item"] = context["results"][0]
+    else:
+        return (len(dedup_set), context)
+    return (1, context)
+
+
+def construct_query(query, size):
+    """
+    Accept some query parameters and construct an OpenSearch query.
+    """
     if not size:
         size = 5
     query = {
@@ -32,7 +185,7 @@ def construct_query(query, fields, size):
         "query": {
             "query_string": {
                 "query": query,
-                "fields": fields,
+                # "fields": fields,
                 # "default_field": "msg",
                 # "type": "best_fields",
                 "fuzziness": "AUTO",
@@ -63,46 +216,3 @@ def construct_query(query, fields, size):
         ],
     }
     return query
-
-
-def filter_blacklisted(user, response):
-    response["redacted"] = 0
-    response["exemption"] = None
-    # For every hit from ES
-    for item in list(response["hits"]["hits"]):
-        # For every blacklisted type
-        for blacklisted_type in settings.OPENSEARCH_BLACKLISTED.keys():
-            # Check this field we are matching exists
-            if blacklisted_type in item["_source"].keys():
-                content = item["_source"][blacklisted_type]
-                # For every item in the blacklisted array for the type
-                for blacklisted_item in settings.OPENSEARCH_BLACKLISTED[
-                    blacklisted_type
-                ]:
-                    if blacklisted_item in str(content):
-                        # Remove the item
-                        if item in response["hits"]["hits"]:
-                            if not user.is_superuser:
-                                response["hits"]["hits"].remove(item)
-                            # Let the UI know something was redacted
-                            response["redacted"] += 1
-                            response["exemption"] = True
-
-
-def run_main_query(client, user, query, fields=None, size=None):
-    if fields:
-        for field in fields:
-            if field not in settings.OPENSEARCH_MAIN_SEARCH_FIELDS:
-                return False
-    if size:
-        if size not in settings.OPENSEARCH_MAIN_SIZES:
-            return False
-    search_query = construct_query(query, fields, size)
-    # fmt: off
-    try:
-        response = client.search(body=search_query,
-                                 index=settings.OPENSEARCH_INDEX_MAIN)
-    except RequestError:
-        return False
-    filter_blacklisted(user, response)
-    return response

core/templates/base.html

@@ -65,7 +65,7 @@
           {% if user.is_authenticated %}
             <div class="navbar-item has-dropdown is-hoverable">
               <a class="navbar-link">
-                Plans
+                Tools
               </a>
       
               <div class="navbar-dropdown">
@@ -73,6 +73,9 @@
                   <a class="navbar-item" href="{% url 'drilldown' %}">
                     Drilldown
                   </a>
+                  <a class="navbar-item" href="{% url 'insights' %}">
+                    Insights
+                  </a>
                 {% endif %}
                 <hr class="navbar-divider">
                 <a class="navbar-item" href="mailto:help@pathogen.is">

core/templates/modals/insights.html

@@ -0,0 +1,132 @@
+{% load index %}
+
+<script>
+  var modal = document.querySelector('.modal');  // assuming you have only 1
+  var html = document.querySelector('html');
+  modal.querySelector('.modal-background').addEventListener('click', function(e) {
+    e.preventDefault();
+    modal.classList.remove('is-active');
+    html.classList.remove('is-clipped');
+  });
+  modal.querySelector('.modal-close').addEventListener('click', function(e) {
+    e.preventDefault();
+    modal.classList.remove('is-active');
+    html.classList.remove('is-clipped');
+  });
+
+  var TABS = [...document.querySelectorAll('#tabs li')];
+  var CONTENT = [...document.querySelectorAll('#tab-content div')];
+  var ACTIVE_CLASS = 'is-active';
+  initTabs();
+</script>
+<style>
+  #tab-content div {
+  display: none;
+  }
+
+  #tab-content div.is-active {
+    display: block;
+  }
+</style>
+<div class="modal is-active is-clipped">
+  <div class="modal-background"></div>
+  <div class="modal-content">
+    <div class="box">
+      <div class="tabs is-toggle is-fullwidth" id="tabs">
+        <ul>
+          <li class="is-active" data-tab="1">
+            <a>
+              <span class="icon is-small"><i class="fa-solid fa-user"></i></span>
+              <span>Channels</span>
+            </a>
+          </li>
+          <li data-tab="2">
+            <a>
+              <span class="icon is-small"><i class="fa-solid fa-hashtag"></i></span>
+              <span>Users</span>
+            </a>
+          </li>
+          <li data-tab="3">
+            <a>
+              <span class="icon is-small"><i class="fa-solid fa-people"></i></span>
+              <span>Intersection</span>
+            </a>
+          </li>
+          <li data-tab="4">
+            <a>
+              <span class="icon is-small"><i class="fa-solid fa-hashtag"></i></span>
+              <span>Intersection</span>
+            </a>
+          </li>
+        </ul>
+      </div>
+      <div id="tab-content">
+        <div class="is-active" data-content="1">
+          <h4 class="subtitle is-4">Channels for {{ nick }} on {{ net }}</h4>
+          {% for channel in chans %}
+            <a class="panel-block">
+              <span class="panel-icon">
+                <i class="fa-solid fa-hashtag" aria-hidden="true"></i>
+              </span>
+              {{ channel }}
+              {% if nick in num_chans %}
+                <span class="tag">
+                  {{ num_users|index:channel }}
+                </span>
+              {% endif %}
+            </a>
+          {% endfor %}
+        </div>
+        <div data-content="2">
+          <h4 class="subtitle is-4">Users on {{ channel }} for {{ net }}</h4>
+          {% for user in users %}
+            <a class="panel-block">
+              <span class="panel-icon">
+                <i class="fa-solid fa-user" aria-hidden="true"></i>
+              </span>
+              {{ user }}
+              {% if channel in num_users %}
+                <span class="tag">
+                  {{ num_chans|index:user }}
+                </span>
+              {% endif %}
+            </a>
+          {% endfor %}
+        </div>
+        <div data-content="3">
+          <h4 class="subtitle is-4">Users sharing channels with {{ nick }} on {{ net }}</h4>
+          {% for user in inter_users %}
+            <a class="panel-block">
+              <span class="panel-icon">
+                <i class="fa-solid fa-user" aria-hidden="true"></i>
+              </span>
+              {{ user }}
+              {% if channel in num_users %}
+                <span class="tag">
+                  {{ num_chans|index:user }}
+                </span>
+              {% endif %}
+            </a>
+          {% endfor %}
+        </div>
+        <div data-content="4">
+          <h4 class="subtitle is-4">Channels sharing users with {{ channel }} on {{ net }}</h4>
+          {% for channel in inter_chans %}
+            <a class="panel-block">
+              <span class="panel-icon">
+                <i class="fa-solid fa-hashtag" aria-hidden="true"></i>
+              </span>
+              {{ channel }}
+              {% if nick in num_chans %}
+                <span class="tag">
+                  {{ num_users|index:channel }}
+                </span>
+              {% endif %}
+            </a>
+          {% endfor %}
+        </div>
+      </div>
+    </div>
+    <button class="modal-close is-large" aria-label="close"></button>
+  </div>
+</div>

core/templates/ui/drilldown/drilldown.html

@@ -56,43 +56,6 @@
       </div>
 
       <div class="columns">
-        <div class="column">
-          <label class="label">Timescale</label>
-          <div class="field-body">
-            <div class="field">
-              <div class="control is-expanded has-icons-left">
-                <div class="select is-fullwidth">
-                  <select name="timescale">
-                    {% for timescale in timescales %}
-                      <option value="{{ timescale }}">{{ timescale }}</option>
-                    {% endfor %}
-                  </select>
-                  <span class="icon is-small is-left">
-                    <i class="fas fa-magnifying-glass"></i>
-                  </span>
-                </div>
-              </div>
-            </div>
-          </div>
-        </div>
-        <div class="column">
-          <label class="label">Fields</label>
-
-          <div class="field">
-            <div class="control is-expanded has-icons-left">
-              <div class="select is-fullwidth is-multiple">
-                <select multiple name="fields">
-                  {% for field in fields %}
-                    <option value="{{ field }}">{{ field }}</option>
-                  {% endfor %}
-                </select>
-                <span class="icon is-small is-left">
-                  <i class="fas fa-magnifying-glass"></i>
-                </span>
-              </div>
-            </div>
-          </div>
-        </div>
         <div class="column">
           <label class="label">Results</label>
           <div class="field">
@@ -114,7 +77,7 @@
 
       <div class="field">
         <div class="control">
-          <button class="button is-primary is-fullwidth" hx-post="{% url 'search' %}" hx-trigger="click" hx-target="#results" hx-swap="outerHTML">
+          <button class="button is-primary is-fullwidth" hx-post="{% url 'search_drilldown' %}" hx-trigger="click" hx-target="#results" hx-swap="outerHTML">
             Search
           </button>
         </div>

core/templates/ui/drilldown/results.html

@@ -1,4 +1,5 @@
 {% load static %}
+{% load index %}
 
 <div id="results">
   {% if results is not None %}
@@ -106,7 +107,7 @@
                   {% if item.src == 'irc' %}
                     <button
                       hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
-                      hx-post="{% url 'modal_info' %}"
+                      hx-post="{% url 'modal_drilldown' %}"
                       hx-vals='{"net": "{{ item.net }}", "nick": "{{ item.nick }}", "channel": "{{ item.channel }}"}'
                       hx-target="#modals-here" 
                       hx-trigger="click"

core/templates/ui/insights/insights.html

@@ -0,0 +1,85 @@
+{% extends "base.html" %}
+{% load static %}
+{% block content %}
+  <script>
+    // tabbed browsing for the modal
+    function initTabs() {
+    TABS.forEach((tab) => {
+      tab.addEventListener('click', (e) => {
+        let selected = tab.getAttribute('data-tab');
+        updateActiveTab(tab);
+        updateActiveContent(selected);
+      })
+    })
+  }
+
+  function updateActiveTab(selected) {
+    TABS.forEach((tab) => {
+      if (tab && tab.classList.contains(ACTIVE_CLASS)) {
+        tab.classList.remove(ACTIVE_CLASS);
+      }
+    });
+    selected.classList.add(ACTIVE_CLASS);
+  }
+
+  function updateActiveContent(selected) {
+    CONTENT.forEach((item) => {
+      if (item && item.classList.contains(ACTIVE_CLASS)) {
+        item.classList.remove(ACTIVE_CLASS);
+      }
+      let data = item.getAttribute('data-content');
+      if (data === selected) {
+        item.classList.add(ACTIVE_CLASS);
+      }
+    });
+  }
+  </script>
+  <style>
+    .icon { border-bottom: 0px !important;}
+  </style>
+  <div class="tile is-ancestor">
+  
+    <div class="tile is-parent is-vertical">
+      <div class="tile is-child box">
+        <form method="POST">
+          {% csrf_token %}
+          <div class="field">
+            <label class="label">Search</label>
+            <div class="field-body">
+              <div class="field">
+                <div class="control is-expanded has-icons-left">
+                  <input name="query" class="input" type="text" placeholder="nickname">
+                  <span class="icon is-small is-left">
+                    <i class="fas fa-magnifying-glass"></i>
+                  </span>
+                </div>
+              </div>
+            </div>
+          </div>
+          <div class="field">
+            <div class="control">
+              <button class="button is-primary is-fullwidth" hx-post="{% url 'search_insights' %}" hx-trigger="click" hx-target="#results" hx-swap="outerHTML">
+                Search
+              </button>
+            </div>
+          </div>
+        </form>
+      </div>
+      
+      <div id="results">
+
+      </div>
+
+    </div>
+  
+    <div class="tile is-parent">
+      <p> 2</p>
+    </div>
+  
+  </div>
+  
+
+
+
+
+{% endblock %}

core/templates/ui/insights/results.html

@@ -0,0 +1,79 @@
+{% load static %}
+{% load index %}
+
+<div class="tile is-child box">
+  <div id="results">
+    {% if item is not None %}
+      <div class="table-container">
+        <table class="table is-hoverable is-fullwidth">
+          <tr>
+            <th>src</th>
+            <td>
+              {% if item|index:'src' == 'irc' %}
+                <span class="icon" data-tooltip="IRC">
+                  <i class="fa-solid fa-hashtag" aria-hidden="true"></i>
+                </span>
+              {% elif item|index:'src' == 'dis' %}
+                <span class="icon" data-tooltip="Discord">
+                  <i class="fa-brands fa-discord" aria-hidden="true"></i>
+                </span>
+              {% endif %}
+            </td>
+          </tr>
+          <tr>
+            <th>nick</th>
+            <td>
+              {% if item|index:'online' is True %}
+                <span class="icon has-text-success has-tooltip-success" data-tooltip="Online">
+                  <i class="fa-solid fa-circle"></i>
+                </span>
+                {{ item.nick }}
+              {% elif item|index:'online' is False %}
+                <span class="icon has-text-danger has-tooltip-danger" data-tooltip="Offline">
+                  <i class="fa-solid fa-circle"></i>
+                </span>
+                {{ item|index:'nick' }}
+              {% else %}
+                <span class="icon has-text-warning has-tooltip-warning" data-tooltip="Unknown">
+                  <i class="fa-solid fa-circle"></i>
+                </span>
+                {{ item|index:'nick'}}
+              {% endif %}
+              {% if item|index:'num_chans' is not None %}
+                <span class="tag">
+                  {{ item|index:'num_chans' }}
+                </span>
+              {% endif %}
+            </td>
+          </tr>
+          <tr>
+            <th>host</th>
+            <td>{{ item.host }}</td>
+          </tr>
+          <tr>
+            <th>actions</th>
+            <td>
+              {% if item.src == 'irc' %}
+                <button
+                  hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
+                  hx-post="{% url 'modal_insights' %}"
+                  hx-vals='{"net": "{{ item.net }}", "nick": "{{ item.nick }}", "channel": "{{ item.channel }}"}'
+                  hx-target="#modals-here" 
+                  hx-trigger="click"
+                  class="button is-small">
+                  Information
+                </button>
+                <div id="modals-here"></div>
+              {% endif %}
+            </td>
+          </tr>
+          <tr>
+            <th>net</th>
+            <td>{{ item.net }}</td>
+          </tr>
+
+        </table>
+      </div>
+    {% endif %}
+  </div>
+</div>

core/ui/views/drilldown.py

@@ -3,21 +3,15 @@ from django.contrib.auth.mixins import LoginRequiredMixin
 from django.shortcuts import render
 from django.views import View
 
-from core.lib.opensearch import initialise_opensearch
-
-client = initialise_opensearch()
-
 
 class Drilldown(LoginRequiredMixin, View):
-    template_name = "ui/drilldown.html"
+    template_name = "ui/drilldown/drilldown.html"
     plan_name = "drilldown"
 
     def get(self, request):
         if not request.user.has_plan(self.plan_name):
             return render(request, "denied.html")
         context = {
-            "fields": settings.OPENSEARCH_MAIN_SEARCH_FIELDS,
             "sizes": settings.OPENSEARCH_MAIN_SIZES,
-            "timescales": settings.OPENSEARCH_MAIN_TIMESCALES,
         }
         return render(request, self.template_name, context)

core/ui/views/insights.py

@@ -0,0 +1,13 @@
+from django.contrib.auth.mixins import LoginRequiredMixin
+from django.shortcuts import render
+from django.views import View
+
+
+class Insights(LoginRequiredMixin, View):
+    template_name = "ui/insights/insights.html"
+    plan_name = "drilldown"
+
+    def get(self, request):
+        if not request.user.has_plan(self.plan_name):
+            return render(request, "denied.html")
+        return render(request, self.template_name)

core/views/dynamic/drilldown.py

@@ -0,0 +1,88 @@
+import json
+
+from django.contrib.auth.mixins import LoginRequiredMixin
+from django.http import HttpResponse, HttpResponseForbidden, JsonResponse
+from django.shortcuts import render
+from django.views import View
+from rest_framework.parsers import FormParser
+from rest_framework.views import APIView
+
+from core.lib.opensearch import query_results
+from core.lib.threshold import (
+    annotate_num_chans,
+    annotate_num_users,
+    get_chans,
+    get_users,
+)
+
+
+class DrilldownSearch(LoginRequiredMixin, View):
+    # parser_classes = [JSONParser]
+    template_name = "ui/drilldown/results.html"
+    plan_name = "drilldown"
+
+    def post(self, request):
+        if not request.user.has_plan(self.plan_name):
+            return HttpResponseForbidden()
+
+        context = query_results(request)
+        if not context:
+            return HttpResponseForbidden()
+        context["data"] = json.dumps(
+            [
+                {
+                    "text": item.get("msg", None) or item.get("id"),
+                    "nick": item.get("nick", None),
+                    "value": item.get("sentiment", None) or None,
+                    "date": item.get("ts"),
+                }
+                for item in context["results"]
+            ]
+        )
+        if context:
+            return render(request, self.template_name, context)
+        else:
+            return HttpResponse("No results")
+
+
+class ThresholdInfoModal(LoginRequiredMixin, APIView):
+    parser_classes = [FormParser]
+    plan_name = "drilldown"
+    template_name = "modals/drilldown.html"
+
+    def post(self, request):
+        if not request.user.has_plan(self.plan_name):
+            return JsonResponse({"success": False})
+        if "net" not in request.data:
+            return JsonResponse({"success": False})
+        if "nick" not in request.data:
+            return JsonResponse({"success": False})
+        if "channel" not in request.data:
+            return JsonResponse({"success": False})
+        net = request.data["net"]
+        nick = request.data["nick"]
+        channel = request.data["channel"]
+        channels = get_chans(net, [nick])
+        users = get_users(net, [channel])
+        num_users = annotate_num_users(net, channels)
+        num_chans = annotate_num_chans(net, users)
+        if channels:
+            inter_users = get_users(net, channels)
+        else:
+            inter_users = []
+        if users:
+            inter_chans = get_chans(net, users)
+        else:
+            inter_chans = []
+        context = {
+            "net": net,
+            "nick": nick,
+            "channel": channel,
+            "chans": channels,
+            "users": users,
+            "inter_chans": inter_chans,
+            "inter_users": inter_users,
+            "num_users": num_users,
+            "num_chans": num_chans,
+        }
+        return render(request, self.template_name, context)

core/views/dynamic/insights.py

@@ -0,0 +1,75 @@
+from django.contrib.auth.mixins import LoginRequiredMixin
+from django.http import HttpResponse, HttpResponseForbidden, JsonResponse
+from django.shortcuts import render
+from django.views import View
+from rest_framework.parsers import FormParser
+from rest_framework.views import APIView
+
+from core.lib.opensearch import query_single_result
+from core.lib.threshold import (
+    annotate_num_chans,
+    annotate_num_users,
+    get_chans,
+    get_users,
+)
+
+
+class InsightsSearch(LoginRequiredMixin, View):
+    # parser_classes = [JSONParser]
+    template_name = "ui/insights/results.html"
+    plan_name = "drilldown"
+
+    def post(self, request):
+        if not request.user.has_plan(self.plan_name):
+            return HttpResponseForbidden()
+
+        results, context = query_single_result(request)
+        if not context:
+            return HttpResponseForbidden()
+        if context:
+            return render(request, self.template_name, context)
+        else:
+            return HttpResponse("No results")
+
+
+class InsightsInfoModal(LoginRequiredMixin, APIView):
+    parser_classes = [FormParser]
+    plan_name = "drilldown"
+    template_name = "modals/insights.html"
+
+    def post(self, request):
+        if not request.user.has_plan(self.plan_name):
+            return JsonResponse({"success": False})
+        if "net" not in request.data:
+            return JsonResponse({"success": False})
+        if "nick" not in request.data:
+            return JsonResponse({"success": False})
+        if "channel" not in request.data:
+            return JsonResponse({"success": False})
+        net = request.data["net"]
+        nick = request.data["nick"]
+        channel = request.data["channel"]
+        channels = get_chans(net, [nick])
+        users = get_users(net, [channel])
+        num_users = annotate_num_users(net, channels)
+        num_chans = annotate_num_chans(net, users)
+        if channels:
+            inter_users = get_users(net, channels)
+        else:
+            inter_users = []
+        if users:
+            inter_chans = get_chans(net, users)
+        else:
+            inter_chans = []
+        context = {
+            "net": net,
+            "nick": nick,
+            "channel": channel,
+            "chans": channels,
+            "users": users,
+            "inter_chans": inter_chans,
+            "inter_users": inter_users,
+            "num_users": num_users,
+            "num_chans": num_chans,
+        }
+        return render(request, self.template_name, context)

core/views/dynamic/search.py

@@ -1,119 +0,0 @@
-import json
-
-from django.conf import settings
-from django.contrib.auth.mixins import LoginRequiredMixin
-from django.http import HttpResponse, HttpResponseForbidden, JsonResponse
-from django.shortcuts import render
-from django.views import View
-
-from core.lib.opensearch import initialise_opensearch, run_main_query
-from core.lib.threshold import annotate_num_chans, annotate_num_users, annotate_online
-
-client = initialise_opensearch()
-
-
-def query_results(request, post_params, api=False):
-    fields = None
-    if "fields" in request.POST:
-        fields = request.POST.getlist("fields")
-    if "size" in request.POST:
-        size = request.POST["size"]
-    if "query" in request.POST:
-        query = request.POST["query"]
-        results = run_main_query(client, request.user, query, fields, size)
-        if not results:
-            return False
-        results_parsed = []
-        if "hits" in results.keys():
-            if "hits" in results["hits"]:
-                for item in results["hits"]["hits"]:
-                    element = item["_source"]
-                    element["id"] = item["_id"]
-                    ts = element["ts"]
-                    ts_spl = ts.split("T")
-                    date = ts_spl[0]
-                    time = ts_spl[1]
-                    element["date"] = date
-                    element["time"] = time
-                    results_parsed.append(element)
-        # Figure out items with net (not discord)
-        nets = set()
-        for x in results_parsed:
-            if "net" in x:
-                nets.add(x["net"])
-
-        for net in nets:
-            # Annotate the online attribute from Threshold
-            online_info = annotate_online(
-                net, [x["nick"] for x in results_parsed if x["src"] == "irc"]
-            )
-            # Annotate the number of users in the channel
-            num_users = annotate_num_users(
-                net, [x["channel"] for x in results_parsed if x["src"] == "irc"]
-            )
-            # Annotate the number channels the user is on
-            num_chans = annotate_num_chans(
-                net, [x["nick"] for x in results_parsed if x["src"] == "irc"]
-            )
-            for item in results_parsed:
-                if item["nick"] in online_info:
-                    item["online"] = online_info[item["nick"]]
-                if item["channel"] in num_users:
-                    item["num_users"] = num_users[item["channel"]]
-                if item["nick"] in num_chans:
-                    item["num_chans"] = num_chans[item["nick"]]
-
-        context = {
-            "query": query,
-            "results": results_parsed,
-            "card": results["hits"]["total"]["value"],
-            "took": results["took"],
-            "redacted": results["redacted"],
-            "exemption": results["exemption"],
-            "fields": settings.OPENSEARCH_MAIN_SEARCH_FIELDS,
-            "sizes": settings.OPENSEARCH_MAIN_SIZES,
-            "timescales": settings.OPENSEARCH_MAIN_TIMESCALES,
-        }
-        return context
-
-
-class Search(LoginRequiredMixin, View):
-    # parser_classes = [JSONParser]
-    template_name = "ui/results.html"
-    plan_name = "drilldown"
-
-    def post(self, request):
-        if not request.user.has_plan(self.plan_name):
-            return HttpResponseForbidden()
-
-        context = query_results(request, request.POST)
-        if not context:
-            return HttpResponseForbidden()
-        context["data"] = json.dumps(
-            [
-                {
-                    "text": item.get("msg", None) or item.get("id"),
-                    "nick": item.get("nick", None),
-                    "value": item.get("sentiment", None) or None,
-                    "date": item.get("ts"),
-                }
-                for item in context["results"]
-            ]
-        )
-        if context:
-            return render(request, self.template_name, context)
-        else:
-            return HttpResponse("No results")
-
-
-class APISearch(LoginRequiredMixin, View):
-    # parser_classes = [JSONParser]
-    template_name = "ui/results.html"
-    plan_name = "drilldown"
-
-    def post(self, request):
-        if not request.user.has_plan(self.plan_name):
-            return JsonResponse({"success": False})
-
-        context = query_results(request, request.POST)
-        return JsonResponse(context)