Improve modal and implement deduplication

core/lib/manage/threshold.py

@@ -196,7 +196,6 @@ def send_irc_message(net, num, channel, msg, nick=None):
     payload = {"msg": msg, "channel": channel}
     if nick:
         payload["nick"] = nick
-    print("SEND", payload)
     messaged = threshold_request(url, payload, method="PUT")
     return messaged
 

core/lib/opensearch.py

@@ -3,6 +3,7 @@ from opensearchpy import OpenSearch
 from opensearchpy.exceptions import NotFoundError, RequestError
 
 from core.lib.threshold import annotate_num_chans, annotate_num_users, annotate_online
+from core.views.helpers import dedup_list
 
 
 def initialise_opensearch():
@@ -49,7 +50,9 @@ def annotate_results(results_parsed):
                 [
                     x["nick"]
                     for x in results_parsed
-                    if x["src"] == "irc" and x["net"] == net and "nick" in x
+                    if {"nick", "src", "net"}.issubset(x)
+                    and x["src"] == "irc"
+                    and x["net"] == net
                 ]
             )
         )
@@ -58,7 +61,9 @@ def annotate_results(results_parsed):
                 [
                     x["channel"]
                     for x in results_parsed
-                    if x["src"] == "irc" and x["net"] == net and "channel" in x
+                    if {"channel", "src", "net"}.issubset(x)
+                    and x["src"] == "irc"
+                    and x["net"] == net
                 ]
             )
         )
@@ -244,7 +249,16 @@ def parse_results(results):
     return results_parsed
 
 
-def query_results(request, query_params, size=None, annotate=True, custom_query=False):
+def query_results(
+    request,
+    query_params,
+    size=None,
+    annotate=True,
+    custom_query=False,
+    reverse=False,
+    dedup=False,
+    dedup_fields=None,
+):
     """
     API helper to alter the OpenSearch return format into something
     a bit better to parse.
@@ -403,8 +417,24 @@ def query_results(request, query_params, size=None, annotate=True, custom_query=
         return {"message": message, "class": message_class}
 
     results_parsed = parse_results(results)
+
     if annotate:
         annotate_results(results_parsed)
+    if "dedup" in query_params:
+        if query_params["dedup"] == "on":
+            dedup = True
+        else:
+            dedup = False
+    else:
+        dedup = False
+
+    if reverse:
+        results_parsed = results_parsed[::-1]
+
+    if dedup:
+        if not dedup_fields:
+            dedup_fields = ["msg", "nick", "ident", "host", "net", "channel"]
+        results_parsed = dedup_list(results_parsed, dedup_fields)
 
     context = {
         "object_list": results_parsed,

core/templates/manage/threshold/irc/network/relays.html

@@ -112,7 +112,8 @@
                              "index": "int",
                              "type": "znc",
                              "mtype": "None",
-                             "nick": "*status"}'
+                             "nick": "*status",
+                             "dedup": "on"}'
                     hx-target="#modals-here" 
                     hx-trigger="click"
                     class="button is-small has-background-info has-text-white">
@@ -201,7 +202,8 @@
                              "index": "int",
                              "type": "auth",
                              "mtype": "None",
-                             "nick": "{{ sinst.entity }}"}'
+                             "nick": "{{ sinst.entity }}",
+                             "dedup": "on"}'
                     hx-target="#modals-here" 
                     hx-trigger="click"
                     class="button is-small has-background-info has-text-white">

core/templates/modals/context_table.html

@@ -1,28 +1,150 @@
 <article class="table-container" id="modal-context-table">
-  <table class="table">
+  <table class="table is-fullwidth">
     <thead>
       <th></th>
+      <th></th>
+      <th></th>
     </thead>
     <tbody>
       {% for item in object_list %}
+        {% if item.type == 'control' %}
+          <tr>
+            <td></td>
+            <td>
+              <span class="icon has-text-grey" data-tooltip="Hidden">
+                <i class="fa-solid fa-file-slash"></i>
+              </span>
+            </td>
+            <td>
+              <p class="has-text-grey">Hidden {{ item.hidden }} similar result{% if item.hidden > 1%}s{% endif %}</p>
+            </td>
+          </tr>
+        {% else %}
           <tr>
             <td>{{ item.time }}</td>
-          {% if query is True and item.type == 'self' %}
+            <td>
-            <td
+              {% if item.type != 'znc' and item.type != 'self' and query is not True %}
-              class="has-tooltip-right {% if item.online is True %}has-text-success{% elif item.online is False %}has-text-danger{% else %}has-text-warning{% endif %}"
+                <article class="nowrap-parent">
-              data-tooltip="You!">{{ item.channel }}</td>
+                  <article class="nowrap-child">
+                    {% if item.type == 'msg' %}
+                      <span class="icon" data-tooltip="Message">
+                        <i class="fa-solid fa-message"></i>
+                      </span>
+                    {% elif item.type == 'join' %}
+                      <span class="icon" data-tooltip="Join">
+                        <i class="fa-solid fa-person-to-portal"></i>
+                      </span>
+                    {% elif item.type == 'part' %}
+                      <span class="icon" data-tooltip="Part">
+                        <i class="fa-solid fa-person-from-portal"></i>
+                      </span>
+                    {% elif item.type == 'quit' %}
+                      <span class="icon" data-tooltip="Quit">
+                        <i class="fa-solid fa-circle-xmark"></i>
+                      </span>
+                    {% elif item.type == 'kick' %}
+                      <span class="icon" data-tooltip="Kick">
+                        <i class="fa-solid fa-user-slash"></i>
+                      </span>
+                    {% elif item.type == 'nick' %}
+                      <span class="icon" data-tooltip="Nick">
+                        <i class="fa-solid fa-signature"></i>
+                      </span>
+                    {% elif item.type == 'mode' %}
+                      <span class="icon" data-tooltip="Mode">
+                        <i class="fa-solid fa-gear"></i>
+                      </span>
+                    {% elif item.type == 'action' %}
+                      <span class="icon" data-tooltip="Action">
+                        <i class="fa-solid fa-exclamation"></i>
+                      </span>
+                    {% elif item.type == 'notice' %}
+                      <span class="icon" data-tooltip="Notice">
+                        <i class="fa-solid fa-message-code"></i>
+                      </span>
+                    {% elif item.type == 'conn' %}
+                      <span class="icon" data-tooltip="Connection">
+                        <i class="fa-solid fa-cloud-exclamation"></i>
+                      </span>
+                    {% elif item.type == 'znc' %}
+                      <span class="icon" data-tooltip="ZNC">
+                        <i class="fa-brands fa-unity"></i>
+                      </span>
+                    {% elif item.type == 'query' %}
+                      <span class="icon" data-tooltip="Query">
+                        <i class="fa-solid fa-message"></i>
+                      </span>
+                    {% elif item.type == 'highlight' %}
+                      <span class="icon" data-tooltip="Highlight">
+                        <i class="fa-solid fa-exclamation"></i>
+                      </span>
+                    {% elif item.type == 'who' %}
+                      <span class="icon" data-tooltip="Who">
+                        <i class="fa-solid fa-passport"></i>
+                      </span>
                     {% else %}
-            <td
+                      {{ item.type }}
-              class="has-tooltip-right {% if item.online is True %}has-text-success{% elif item.online is False %}has-text-danger{% else %}has-text-warning{% endif %}"
-              data-tooltip="{{ item.nick }}!{{ item.ident }}@{{ item.host }}">{{ item.nick }}</td>
                     {% endif %}
+                    {% if item.online is True %}
+                      <span class="icon has-text-success has-tooltip-success" data-tooltip="Online">
+                        <i class="fa-solid fa-circle"></i>
+                      </span>
+                    {% elif item.online is False %}
+                      <span class="icon has-text-danger has-tooltip-danger" data-tooltip="Offline">
+                        <i class="fa-solid fa-circle"></i>
+                      </span>
+                    {% else %}
+                      <span class="icon has-text-warning has-tooltip-warning" data-tooltip="Unknown">
+                        <i class="fa-solid fa-circle"></i>
+                      </span>
+                    {% endif %}
+                    {% if item.src == 'irc' %}
+                      <a
+                        hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
+                        hx-post="{% url 'modal_drilldown' %}"
+                        hx-vals='{"net": "{{ item.net }}", "nick": "{{ item.nick }}", "channel": "{{ item.channel }}"}'
+                        hx-target="#modals-here"
+                        hx-trigger="click"
+                        class="has-text-black">
+                        <span class="icon" data-tooltip="Open drilldown modal">
+                          <i class="fa-solid fa-album"></i>
+                        </span>
+                      </a>
+                    {% endif %}
+                  </article>
+                  <a class="nowrap-child has-text-link is-underlined" onclick="populateSearch('nick', '{{ item.nick|escapejs }}')">
+                    {{ item.nick }}
+                  </a>
+                  {% if item.num_chans != '—' %}
+                    <article class="nowrap-child">
+                      <span class="tag">
+                        {{ item.num_chans }}
+                      </span>
+                    </article>
+                  {% endif %}
+                </article>
+              {% endif %}
+              {% if item.type == 'self' %}
+                <span class="icon has-text-primary" data-tooltip="You">
+                  <i class="fa-solid fa-message-check"></i>
+                </span>
+              {% elif item.type == 'znc' %}
+                <span class="icon has-text-info" data-tooltip="ZNC">
+                  <i class="fa-brands fa-unity"></i>
+                </span>
+              {% elif query %}
+                <span class="icon has-text-info" data-tooltip="Auth">
+                  <i class="fa-solid fa-passport"></i>
+                </span>
+              {% endif %}
+            </td>
             <td class="wrap">{{ item.msg }}</td>
           </tr>
+        {% endif %}
       {% endfor %}
     </tbody>
   </table>
-</article>
+  {% if object_list %}
-{% if object_list %}
     <div
       class="modal-refresh"
       style="display: none;"
@@ -37,11 +159,14 @@
                "index": "{{ index }}",
                "type": "{{ type }}",
                "mtype": "{{ mtype }}",
-             "nick": "{{ nick }}"}'
+               "nick": "{{ nick }}",
+               "dedup": "{{ params.dedup }}"}'
       hx-target="#modal-context-table"
       hx-trigger="every 5s">
     </div>
-{% endif %}
+  {% endif %}
+</article>
+
 <script>
   var modal_event = new Event('restore-modal-scroll');
   document.dispatchEvent(modal_event);

core/templates/ui/drilldown/drilldown.html

@@ -179,7 +179,7 @@
             </div>
           </div>
           <div class="column is-narrow">
-            <div class="field has-addons">
+            <div class="field has-addons block">
               <div class="control has-icons-left">
                 <span class="select">
                   <select id="source" name="source">
@@ -215,6 +215,18 @@
                 </a>
               </p>
             </div>
+            <div class="control">
+              <label class="button has-text-link">
+                <input type="checkbox"
+                       name="dedup"
+                       {% if params.dedup == "on" %}
+                         checked
+                       {% endif %}>
+                <span class="icon" data-tooltip="Deduplicate results">
+                  <i class="fa-solid fa-copy"></i>
+                </span>
+              </label>
+            </div>
           </div>
           <div class="column is-narrow">
             <div id="sentiment block">

core/templates/ui/drilldown/table_results_partial.html

@@ -111,6 +111,19 @@
             <tbody {{ table.attrs.tbody.as_html }}>
               {% for row in table.paginated_rows %}
                 {% block table.tbody.row %}
+                  {% if row.cells.type == 'control' %}
+                    <tr>
+                      <td></td>
+                      <td>
+                        <span class="icon has-text-grey" data-tooltip="Hidden">
+                          <i class="fa-solid fa-file-slash"></i>
+                        </span>
+                      </td>
+                      <td>
+                        <p class="has-text-grey">Hidden {{ row.cells.hidden }} similar result{% if row.cells.hidden > 1%}s{% endif %}</p>
+                      </td>
+                    </tr>
+                  {% else %}
                     <tr class="
                                {% if row.cells.exemption == True %}has-background-grey-lighter
                                {% elif cell == 'join' %}has-background-success-light
@@ -134,7 +147,8 @@
                               </td>
                             {% elif column.name == 'src' %}
                               <td class="{{ column.name }}">
-                              <a class="has-text-link is-underlined"
+                                <a 
+                                  class="has-text-link is-underlined"
                                   onclick="populateSearch('src', '{{ cell|escapejs }}')">
                                   {% if row.cells.src == 'irc' %}
                                     <span class="icon" data-tooltip="IRC">
@@ -154,7 +168,8 @@
                               </td>
                             {% elif column.name == 'type' or column.name == 'mtype' %}
                               <td class="{{ column.name }}">
-                              <a class="has-text-link is-underlined"
+                                <a 
+                                  class="has-text-link is-underlined"
                                   onclick="populateSearch('{{ column.name }}', '{{ cell|escapejs }}')">
                                   {% if cell == 'msg' %}
                                     <span class="icon" data-tooltip="Message">
@@ -232,7 +247,8 @@
                                            "index": "{{ params.index }}",
                                            "type": "{{ row.cells.type }}",
                                            "mtype": "{{ row.cells.mtype }}",
-                                         "nick": "{{ row.cells.nick|escapejs }}"}'
+                                           "nick": "{{ row.cells.nick|escapejs }}",
+                                           "dedup": "{{ params.dedup }}"}'
                                   hx-target="#modals-here" 
                                   hx-trigger="click"
                                   href="/?modal=context&net={{row.cells.net|escapejs}}&num={{row.cells.num|escapejs}}&src={{row.cells.src|escapejs}}&channel={{row.cells.channel|urlsafe}}&time={{row.cells.time|escapejs}}&date={{row.cells.date|escapejs}}&index={{params.index}}&type={{row.cells.type}}&mtype={{row.cells.mtype}}&nick={{row.cells.mtype|escapejs}}">
@@ -286,7 +302,8 @@
                               <td class="{{ column.name }}">
                                 {% if cell != '—' %}
                                   <div class="nowrap-parent">
-                                  <a class="nowrap-child has-text-link is-underlined"
+                                    <a 
+                                      class="nowrap-child has-text-link is-underlined"
                                       onclick="populateSearch('channel', '{{ cell|escapejs }}')">
                                       {{ cell }}
                                     </a>
@@ -316,7 +333,8 @@
                               </td>
                             {% else %}
                               <td class="{{ column.name }}">
-                              <a class="has-text-link is-underlined"
+                                <a 
+                                  class="has-text-link is-underlined"
                                   onclick="populateSearch('{{ column.name }}', '{{ cell|escapejs }}')">
                                   {{ cell }}
                                 </a>
@@ -326,6 +344,7 @@
                         {% endif %}
                       {% endfor %}
                     </tr>
+                  {% endif %}
                 {% endblock table.tbody.row %}
               {% empty %}
                 {% if table.empty_text %}

core/views/helpers.py

@@ -0,0 +1,39 @@
+def dedup_list(data, check_keys):
+    """
+    Remove duplicate dictionaries from list.
+    """
+    seen = set()
+    out = []
+
+    dup_count = 0
+    for x in data:
+        dedupeKey = tuple(x[k] for k in check_keys if k in x)
+        if dedupeKey in seen:
+            dup_count += 1
+            continue
+        if dup_count > 0:
+            out.append({"type": "control", "hidden": dup_count})
+            dup_count = 0
+        out.append(x)
+        seen.add(dedupeKey)
+    if dup_count > 0:
+        out.append({"type": "control", "hidden": dup_count})
+    return out
+
+
+
+
+
+# from random import randint
+# from timeit import timeit
+# entries = 10000
+# a = [
+#     {'ts': "sss", 'msg': randint(1, 2), str(randint(1, 2)): randint(1, 2)} for x in range(entries)
+#     ]
+# kk = ["msg", "nick"]
+# call = lambda: dedup_list(a, kk)
+# #print(timeit(call, number=10))
+# print(dedup_list(a, kk))
+
+# # sh-5.1$ python helpers.py 
+# # 1.0805372429895215

core/views/ui/drilldown.py

@@ -254,14 +254,11 @@ class DrilldownContextModal(APIView):
         nicks = None
         query = False
         # Create the query params from the POST arguments
-        mandatory = ["net", "channel", "num", "src", "index", "nick", "type"]
+        mandatory = ["net", "channel", "num", "src", "index", "nick", "type", "mtype"]
         invalid = [None, False, "—", "None"]
 
         query_params = {k: v for k, v in request.data.items() if v}
 
-        if query_params["index"] == "int":
-            mandatory.append("mtype")
-
         for key in query_params:
             if query_params[key] in invalid:
                 query_params[key] = None
@@ -314,13 +311,16 @@ class DrilldownContextModal(APIView):
         if query_params["src"] == "irc":
             if query_params["type"] in ["query", "notice", "msg", "highlight"]:
                 annotate = True
-
         results = query_results(
-            request, query_params, annotate=annotate, custom_query=search_query
+            request,
+            query_params,
+            annotate=annotate,
+            custom_query=search_query,
+            reverse=True,
+            dedup_fields=["net", "type", "msg"],
         )
         if "message" in results:
             return render(request, self.template_name, results)
-        results["object_list"] = results["object_list"][::-1]
 
         # Make the time nicer
         # for index, item in enumerate(results["object_list"]):
@@ -339,6 +339,7 @@ class DrilldownContextModal(APIView):
             "type": query_params["type"],
             "mtype": query_params["mtype"],
             "nick": query_params["nick"],
+            "params": query_params,
         }
         if request.user.is_superuser:
             if query:

core/views/ui/tables.py

@@ -66,5 +66,6 @@ class DrilldownTable(Table):
     server = Column()
     mtype = Column()
     tokens = Column()
+    hidden = Column()
     template_name = "ui/drilldown/table_results.html"
     paginate_by = settings.DRILLDOWN_RESULTS_PER_PAGE