Implement scrollback modal

core/lib/context.py

@@ -0,0 +1,31 @@
+from core.lib.opensearch import client, run_main_query
+from django.conf import settings
+
+def construct_query(net, channel, src, num, size):
+    # Get the initial query
+    extra_must = []
+    if num:
+        extra_must.append({"match": {"num": num}})
+    types = ["msg", "notice", "action", "kick", "topic", "mode"]
+    query_should = [{"match": {"type": x}} for x in types]
+    query = {
+        "size": size,
+        "query": {
+            "bool": {
+                "must": [
+                    {"match": {"net": net}},
+                    {"match": {"channel": channel}},
+                    {"match": {"src": src}},
+                    {
+                        "bool": {
+                            "should": query_should,
+                        }
+                    },
+                    *extra_must
+                ]
+            }
+        },
+        "fields": ["nick", "ident", "host", "channel", "ts", "msg", "type", "net", "src"],
+        "_source": False,
+    }
+    return query

core/lib/opensearch.py

@@ -97,8 +97,14 @@ def filter_blacklisted(user, response):
         # For every blacklisted type
         for blacklisted_type in settings.OPENSEARCH_BLACKLISTED.keys():
             # Check this field we are matching exists
-            if blacklisted_type in item["_source"].keys():
+            if "_source" in item.keys():
-                content = item["_source"][blacklisted_type]
+                data_index = "_source"
+            elif "fields" in item.keys():
+                data_index = "fields"
+            else:
+                return False
+            if blacklisted_type in item[data_index].keys():
+                content = item[data_index][blacklisted_type]
                 # For every item in the blacklisted array for the type
                 for blacklisted_item in settings.OPENSEARCH_BLACKLISTED[
                     blacklisted_type
@@ -109,7 +115,7 @@ def filter_blacklisted(user, response):
                             # Let the UI know something was redacted
                             if (
                                 "exemption"
-                                not in response["hits"]["hits"][index]["_source"]
+                                not in response["hits"]["hits"][index][data_index]
                             ):
                                 response["redacted"] += 1
                             # Anonymous
@@ -120,7 +126,7 @@ def filter_blacklisted(user, response):
                                 if not user.is_superuser:
                                     response["hits"]["hits"][index] = None
                                 else:
-                                    response["hits"]["hits"][index]["_source"][
+                                    response["hits"]["hits"][index][data_index][
                                         "exemption"
                                     ] = True
 
@@ -128,6 +134,49 @@ def filter_blacklisted(user, response):
     response["hits"]["hits"] = [hit for hit in response["hits"]["hits"] if hit]
 
 
+def construct_query(query, size):
+    """
+    Accept some query parameters and construct an OpenSearch query.
+    """
+    if not size:
+        size = 5
+    query = {
+        "size": size,
+        "query": {
+            "bool": {
+                "must": [
+                    {
+                        "query_string": {
+                            "query": query,
+                            # "fields": fields,
+                            # "default_field": "msg",
+                            # "type": "best_fields",
+                            "fuzziness": "AUTO",
+                            "fuzzy_transpositions": True,
+                            "fuzzy_max_expansions": 50,
+                            "fuzzy_prefix_length": 0,
+                            # "minimum_should_match": 1,
+                            "default_operator": "or",
+                            "analyzer": "standard",
+                            "lenient": True,
+                            "boost": 1,
+                            "allow_leading_wildcard": True,
+                            # "enable_position_increments": False,
+                            "phrase_slop": 3,
+                            # "max_determinized_states": 10000,
+                            "quote_field_suffix": "",
+                            "quote_analyzer": "standard",
+                            "analyze_wildcard": False,
+                            "auto_generate_synonyms_phrase_query": True,
+                        }
+                    }
+                ]
+            }
+        },
+    }
+    return query
+
+
 def run_main_query(client, user, query, custom_query=False, index=None, size=None):
     """
     Low level helper to run an ES query.
@@ -153,12 +202,22 @@ def run_main_query(client, user, query, custom_query=False, index=None, size=Non
     filter_blacklisted(user, response)
     return response
 
+
 def parse_results(results):
     results_parsed = []
     if "hits" in results.keys():
         if "hits" in results["hits"]:
             for item in results["hits"]["hits"]:
-                element = item["_source"]
+                if "_source" in item.keys():
+                    data_index = "_source"
+                elif "fields" in item.keys():
+                    data_index = "fields"
+                else:
+                    return False
+                element = item[data_index]
+                # Why are fields in lists...
+                if data_index == "fields":
+                    element = {k: v[0] for k, v in element.items() if len(v)}
                 element["id"] = item["_id"]
 
                 # Split the timestamp into date and time
@@ -173,11 +232,19 @@ def parse_results(results):
                     date = ts_spl[0]
                     time = ts_spl[1]
                     element["date"] = date
-                    element["time"] = time
+                    if "." in time:
+                        time_spl = time.split(".")
+                        if len(time_spl) == 2:
+                            element["time"] = time.split(".")[0]
+                        else:
+                            element["time"] = time
+                    else:
+                        element["time"] = time
                 results_parsed.append(element)
     return results_parsed
 
-def query_results(request, query_params, size=None):
+
+def query_results(request, query_params, size=None, annotate=True, custom_query=False):
     """
     API helper to alter the OpenSearch return format into something
     a bit better to parse.
@@ -185,6 +252,7 @@ def query_results(request, query_params, size=None):
     results with the other data we have.
     """
     # is_anonymous = isinstance(request.user, AnonymousUser)
+    query = None
     message = None
     message_class = None
     add_bool = []
@@ -202,6 +270,8 @@ def query_results(request, query_params, size=None):
                 message = "Size is not permitted"
                 message_class = "danger"
                 return {"message": message, "class": message_class}
+        else:
+            size = 20
     if "source" in query_params:
         source = query_params["source"]
         if source not in settings.OPENSEARCH_MAIN_SOURCES:
@@ -275,72 +345,79 @@ def query_results(request, query_params, size=None):
     if "query" in query_params:
         query = query_params["query"]
         search_query = construct_query(query, size)
-        if add_bool:
+    else:
-            for item in add_bool:
+        print("NO QUERY")
-                search_query["query"]["bool"]["must"].append({"match": item})
+        if custom_query:
-        if add_top:
+            print("CUSTOM")
-            for item in add_top:
+            search_query = custom_query
-                search_query["query"]["bool"]["must"].append(item)
+    if add_bool:
-        if add_top_negative:
+        for item in add_bool:
-            for item in add_top_negative:
+            search_query["query"]["bool"]["must"].append({"match": item})
-                if "must_not" in search_query["query"]["bool"]:
+    if add_top:
-                    search_query["query"]["bool"]["must_not"].append(item)
+        for item in add_top:
-                else:
+            search_query["query"]["bool"]["must"].append(item)
-                    search_query["query"]["bool"]["must_not"] = [item]
+    if add_top_negative:
-        if sort:
+        for item in add_top_negative:
-            search_query["sort"] = sort
+            if "must_not" in search_query["query"]["bool"]:
-
+                search_query["query"]["bool"]["must_not"].append(item)
-        if "index" in query_params:
-            if not request.user.is_superuser:
-                message = "How did you get here?"
-                message_class = "danger"
-                return {"message": message, "class": message_class}
             else:
-                index = query_params["index"]
+                search_query["query"]["bool"]["must_not"] = [item]
-                if index == "main":
+    if sort:
-                    index = settings.OPENSEARCH_INDEX_MAIN
+        search_query["sort"] = sort
-                elif index == "meta":
+
+    if "index" in query_params:
+        index = query_params["index"]
+        if index == "main":
+            index = settings.OPENSEARCH_INDEX_MAIN
+        else:
+            if request.user.is_superuser:
+                if index == "meta":
                     index = settings.OPENSEARCH_INDEX_META
-                elif index == "int":
+                if index == "int":
                     index = settings.OPENSEARCH_INDEX_INT
                 else:
                     message = "Index is not valid."
                     message_class = "danger"
                     return {"message": message, "class": message_class}
-        else:
+            else:
-            index = settings.OPENSEARCH_INDEX_MAIN
+                message = "Not permitted to search by this index"
-        results = run_main_query(
+                message_class = "danger"
-            client,
+                return {"message": message, "class": message_class}
-            request.user,  # passed through run_main_query to filter_blacklisted
+    else:
-            search_query,
+        index = settings.OPENSEARCH_INDEX_MAIN
-            custom_query=True,
+    results = run_main_query(
-            index=index,
+        client,
-            size=size,
+        request.user,  # passed through run_main_query to filter_blacklisted
-        )
+        search_query,
-        if not results:
+        custom_query=True,
-            return False
+        index=index,
-        if isinstance(results, Exception):
+        size=size,
-            message = results.info["error"]["root_cause"][0]["reason"]
+    )
-            message_class = "danger"
+    if not results:
-            return {"message": message, "class": message_class}
+        return False
-        if len(results["hits"]["hits"]) == 0:
+    if isinstance(results, Exception):
-            message = "No results."
+        message = results.info["error"]["root_cause"][0]["reason"]
-            message_class = "danger"
+        message_class = "danger"
-            return {"message": message, "class": message_class}
+        return {"message": message, "class": message_class}
-
+    if len(results["hits"]["hits"]) == 0:
-        results_parsed = parse_results(results)
+        message = "No results."
+        message_class = "danger"
+        return {"message": message, "class": message_class}
 
+    results_parsed = parse_results(results)
+    if annotate:
         annotate_results(results_parsed)
 
-        context = {
+    context = {
-            "query": query,
+        "object_list": results_parsed,
-            "object_list": results_parsed,
+        "card": results["hits"]["total"]["value"],
-            "card": results["hits"]["total"]["value"],
+        "took": results["took"],
-            "took": results["took"],
+        "redacted": results["redacted"],
-            "redacted": results["redacted"],
+        "exemption": results["exemption"],
-            "exemption": results["exemption"],
+    }
-        }
+    if query:
-        return context
+        context["query"] = query
+    return context
 
 
 def query_single_result(request):
@@ -355,46 +432,3 @@ def query_single_result(request):
         context["item"] = context["object_list"][0]
 
     return context
-
-
-def construct_query(query, size):
-    """
-    Accept some query parameters and construct an OpenSearch query.
-    """
-    if not size:
-        size = 5
-    query = {
-        "size": size,
-        "query": {
-            "bool": {
-                "must": [
-                    {
-                        "query_string": {
-                            "query": query,
-                            # "fields": fields,
-                            # "default_field": "msg",
-                            # "type": "best_fields",
-                            "fuzziness": "AUTO",
-                            "fuzzy_transpositions": True,
-                            "fuzzy_max_expansions": 50,
-                            "fuzzy_prefix_length": 0,
-                            # "minimum_should_match": 1,
-                            "default_operator": "or",
-                            "analyzer": "standard",
-                            "lenient": True,
-                            "boost": 1,
-                            "allow_leading_wildcard": True,
-                            # "enable_position_increments": False,
-                            "phrase_slop": 3,
-                            # "max_determinized_states": 10000,
-                            "quote_field_suffix": "",
-                            "quote_analyzer": "standard",
-                            "analyze_wildcard": False,
-                            "auto_generate_synonyms_phrase_query": True,
-                        }
-                    }
-                ]
-            }
-        },
-    }
-    return query

core/static/js/column-shifter.js

@@ -122,7 +122,7 @@ $(document).ready(function(){
     // Show table content and hide spiner
     var show_table_content = function($table_class_container){
         $table_class_container.find("#loader").hide();
-        $table_class_container.find(".table-container").show();
+        $table_class_container.find("#table-container").show();
         const event = new Event('restore-scroll');
         document.dispatchEvent(event);
     };
@@ -222,7 +222,7 @@ $(document).ready(function(){
         if(idx==undefined) {
             idx = 0;
         }
-        return $('.table-container').eq(idx).find('.btn-shift-column').filter(function(z) {
+        return $('#table-container').eq(idx).find('.btn-shift-column').filter(function(z) {
             return $(this).data('state')=='off'
         }).map(function(z) { 
             return $(this).data('td-class')

core/static/modal.js

@@ -2,12 +2,20 @@
 var modal = document.getElementById("modal");
 var html = document.querySelector('html');
 
+var disableModal = function() {
+  modal.classList.remove('is-active');
+  html.classList.remove('is-clipped');
+  var modal_refresh = document.getElementsByClassName("modal-refresh");
+  for(var i = 0; i < modal_refresh.length; i++) {
+    modal_refresh[i].remove();
+}
+}
+
 var elements = document.querySelectorAll('.modal-background');
 for(var i = 0; i < elements.length; i++) {
     elements[i].addEventListener('click', function(e) {
       // elements[i].preventDefault();
-      modal.classList.remove('is-active');
+      disableModal();
-      html.classList.remove('is-clipped');
     });
 }
 
@@ -15,8 +23,7 @@ var elements = document.querySelectorAll('.modal-close');
 for(var i = 0; i < elements.length; i++) {
     elements[i].addEventListener('click', function(e) {
       // elements[i].preventDefault();
-      modal.classList.remove('is-active');
+      disableModal();
-      html.classList.remove('is-clipped');
     });
 }
 
@@ -25,8 +32,7 @@ function activateButtons() {
   for(var i = 0; i < elements.length; i++) {
       elements[i].addEventListener('click', function(e) {
         // elements[i].preventDefault();
-        modal.classList.remove('is-active');
+        disableModal();
-        html.classList.remove('is-clipped');
       });
   }
 }

core/templates/modals/context.html

@@ -4,10 +4,44 @@
 <script src="{% static 'modal.js' %}"></script>
 <link rel ="stylesheet" href="{% static 'tabs.css' %}">
 <script src="{% static 'tabs.js' %}"></script>
+
+<div
+  class="modal-refresh"
+  style="display: none;"
+  hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
+  hx-post="{% url 'modal_context' %}"
+  hx-vals='{"net": "{{ net }}",
+            "num": "{{ num }}",
+            "src": "{{ src }}",
+            "channel": "{{ channel }}",
+            "time": "{{ time }}",
+            "date": "{{ date }}",
+            "index": "{{ index }}"
+          }'
+  hx-target="#modals-here"
+  hx-trigger="every 3s">
+</div>
+
+<script>
+  document.addEventListener("restore-modal-scroll", function(event) {
+      var scrollpos = localStorage.getItem('scrollpos_modal_content');
+      if (scrollpos) {
+        document.getElementsByClassName("modal-content")[0].scrollTop = scrollpos;
+      };
+  });
+
+  document.addEventListener("htmx:beforeSwap", function(event) {
+    var contentScroll = document.getElementsByClassName("modal-content")[0].scrollTop;
+    localStorage.setItem('scrollpos_modal_content', contentScroll);
+
+  });
+</script>
+
 <div id="modal" class="modal is-active is-clipped">
   <div class="modal-background"></div>
   <div class="modal-content">
     <div class="box">
+      {% include 'partials/notify.html' %}
       <div class="tabs is-toggle is-fullwidth is-info" id="tabs">
         <ul>
           <li class="is-active" data-tab="1">
@@ -39,7 +73,52 @@
       <div id="tab-content">
         <div class="is-active" data-content="1">
           <h4 class="subtitle is-4">Scrollback of {{ channel }} on {{ net }}{{ num }}</h4>
-          scrollback
+          <article class="table-container">
+            <table class="table">
+              <thead>
+                <th></th>
+              </thead>
+              <tbody>
+                {% for item in object_list %}
+                  {% if item.type == 'msg' %}
+                    <tr>
+                      <td>{{ item.time }}</td>
+                      <td
+                        class="has-tooltip-right {% if item.online is True %}has-text-success{% elif item.online is False %}has-text-danger{% else %}has-text-warning{% endif %}"
+                        data-tooltip="{{ item.nick }}!{{ item.ident }}@{{ item.host }}">{{ item.nick }}</td>
+                      <td class="wrap">{{ item.msg }}</td>
+                    </tr>
+                  {% endif %}
+                {% endfor %}
+              </tbody>
+            </table>
+          </article>
+          <div class="field has-addons">
+            <div class="control is-expanded has-icons-left">
+              <input
+                hx-post="{% url 'search' %}"
+                hx-trigger="keyup changed delay:200ms"
+                hx-target="#results"
+                hx-swap="innerHTML" id="query" name="query" value="{{ params.query }}" class="input" type="text" placeholder="msg: science AND nick: BillNye AND channel: #science">
+              <span class="icon is-small is-left">
+                <i class="fas fa-magnifying-glass"></i>
+              </span>
+            </div>
+            <div class="control">
+              <div class="field">
+                <button
+                  id="search"
+                  class="button is-info is-fullwidth"
+                  hx-post="{% url 'search' %}"
+                  hx-trigger="click"
+                  hx-target="#results"
+                  hx-swap="innerHTML">
+                  Search
+                </button>
+              </div>
+            </div>
+          </div>
+
         </div>
         <div data-content="2">
           <h4 class="subtitle is-4">Scrollback of {{ channel }} on {{ net }}{{ num }} around {{ ts }}</h4>
@@ -56,6 +135,10 @@
       </div>
     </div>
     <button class="modal-close is-large" aria-label="close"></button>
+    <script>
+      var modal_event = new Event('restore-modal-scroll');
+      document.dispatchEvent(modal_event);
+    </script>
     {# <script>activateButtons();</script> #}
   </div>
 </div>

core/templates/ui/drilldown/drilldown.html

@@ -2,6 +2,22 @@
 {% load static %}
 {% load joinsep %}
 {% block content %}
+  {% if params.modal == 'context' %}
+    <div style="display: none;"
+    hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
+    hx-post="{% url 'modal_context' %}"
+    hx-vals='{"net": "{{ params.net|escapejs }}",
+              "num": "{{ params.num|escapejs }}",
+              "src": "{{ params.src|escapejs }}",
+              "channel": "{{ params.channel|escapejs }}",
+              "time": "{{ params.time|escapejs }}",
+              "date": "{{ params.date|escapejs }}",
+              "index": "{{ params.index }}"
+            }'
+    hx-target="#modals-here" 
+    hx-trigger="load">
+  </div>
+  {% endif %}
   <script src="{% static 'js/chart.js' %}"></script>
   <script>
     function setupTags() {

core/templates/ui/drilldown/table_results.html

@@ -2,6 +2,7 @@
 {% load index %}
 {% load joinsep %}
 {% include 'partials/notify.html' %}
+
 {% if table %}
   <script src="{% static 'js/column-shifter.js' %}"></script>
   <div style="display: none" id="jsonData" data-json="{{ data }}">
@@ -26,7 +27,7 @@
       {% endif %}
     {% endif %}
   </div>
-  {% if params.index == 'main' %}
+  {% if params.index != 'int' and params.index != 'meta' %}
     <div class="box">
       <div style="height: 30rem">
         <canvas id="volume"></canvas>
@@ -34,6 +35,7 @@
       <script src="{% static 'chart.js' %}"></script>
     </div>
   {% endif %}
+  {{ params }}
   {% include 'ui/drilldown/table_results_partial.html' %}
 {% endif %}
 {# Update the tags in case the user changed the query #}

core/templates/ui/drilldown/table_results_partial.html

@@ -52,7 +52,7 @@
         });
 
       </script>
-      <div class="table-container" style="display:none;">
+      <div id="table-container" class="table-container" style="display:none;">
         <table {% render_attrs table.attrs class="table is-striped is-hoverable is-fullwidth" %}>
           {% block table.thead %}
             {% if table.show_header %}
@@ -219,10 +219,10 @@
                                  hx-vals='{"net": "{{ row.cells.net|escapejs }}",
                                            "num": "{{ row.cells.num|escapejs }}",
                                            "src": "{{ row.cells.src|escapejs }}",
-                                           "nick": "{{ row.cells.nick|escapejs }}",
                                            "channel": "{{ row.cells.channel|escapejs }}",
                                            "time": "{{ row.cells.time|escapejs }}",
-                                           "date": "{{ row.cells.date|escapejs }}"
+                                           "date": "{{ row.cells.date|escapejs }}",
+                                           "index": "{{ params.index }}"
                                           }'
                                  hx-target="#modals-here" 
                                  hx-trigger="click">

core/views/ui/drilldown.py

@@ -10,6 +10,7 @@ from django_tables2 import SingleTableView
 from rest_framework.parsers import FormParser
 from rest_framework.views import APIView
 
+from core.lib.context import construct_query
 from core.lib.opensearch import query_results
 from core.lib.threshold import (
     annotate_num_chans,
@@ -114,10 +115,13 @@ def drilldown_search(request, return_context=False, template=None):
             query_params["from_time"] = dates["from_time"]
             query_params["to_time"] = dates["to_time"]
 
-    if request.GET:
+    if "query" in query_params:
-        context = query_results(request, query_params)
+        if request.GET:
-    elif request.POST:
+            context = query_results(request, query_params)
-        context = query_results(request, query_params)
+        elif request.POST:
+            context = query_results(request, query_params)
+    else:
+        context = {"object_list": []}
 
     # URI we're passing to the template for linking
     if "csrfmiddlewaretoken" in query_params:
@@ -238,7 +242,10 @@ class DrilldownContextModal(APIView):
     def post(self, request):
         # if not request.user.has_plan(self.plan_name):
         #    return JsonResponse({"success": False})
-        num = ""
+        num = None
+        index = "main"
+        size = 10
+        # Check right fields exist
         if "net" not in request.data:
             return JsonResponse({"success": False})
         if "channel" not in request.data:
@@ -250,17 +257,49 @@ class DrilldownContextModal(APIView):
         if "date" not in request.data:
             return JsonResponse({"success": False})
         if "num" in request.data:
-            num = request.data["num"]
+            if request.data["num"]:
+                if not request.data["num"] == "—":
+                    num = request.data["num"]
+        if "index" in request.data:
+            if request.data["index"]:
+                index = request.data["index"]
         net = request.data["net"]
         channel = request.data["channel"]
-        # results = 
+        # Create the query params from the POST arguments
+        blacklisted = ["date", "time"]
+        query_params = {
+            k: v for k, v in request.data.items() if v and k not in blacklisted
+        }
+        query_params["sorting"] = "desc"
+
+        # Create the query with the context helper
+        search_query = construct_query(net, channel, request.data["src"], num, size)
+        print("SEARCH QUERY", search_query)
+        results = query_results(
+            request, query_params, annotate=True, custom_query=search_query
+        )
+        if "message" in results:
+            return render(request, self.template_name, results)
+        results["object_list"] = results["object_list"][::-1]
+
+        # Make the time nicer
+        # for index, item in enumerate(results["object_list"]):
+        #    results["object_list"][index]["time"] = item["time"]+"SSS"
+
         context = {
             "net": net,
             "channel": channel,
-            "num": num,
             "src": request.data["src"],
             "ts": f"{request.data['date']} {request.data['time']}",
+            "object_list": results["object_list"],
+            "time": request.data["time"],
+            "date": request.data["date"],
+            "index": request.data["date"],
         }
+        if index:
+            context["index"] = index
+        if num:
+            context["num"] = num
         return render(request, self.template_name, context)