Improve modal and implement deduplication

core/views/helpers.py

@@ -0,0 +1,39 @@
+def dedup_list(data, check_keys):
+    """
+    Remove duplicate dictionaries from list.
+    """
+    seen = set()
+    out = []
+
+    dup_count = 0
+    for x in data:
+        dedupeKey = tuple(x[k] for k in check_keys if k in x)
+        if dedupeKey in seen:
+            dup_count += 1
+            continue
+        if dup_count > 0:
+            out.append({"type": "control", "hidden": dup_count})
+            dup_count = 0
+        out.append(x)
+        seen.add(dedupeKey)
+    if dup_count > 0:
+        out.append({"type": "control", "hidden": dup_count})
+    return out
+
+
+
+
+
+# from random import randint
+# from timeit import timeit
+# entries = 10000
+# a = [
+#     {'ts': "sss", 'msg': randint(1, 2), str(randint(1, 2)): randint(1, 2)} for x in range(entries)
+#     ]
+# kk = ["msg", "nick"]
+# call = lambda: dedup_list(a, kk)
+# #print(timeit(call, number=10))
+# print(dedup_list(a, kk))
+
+# # sh-5.1$ python helpers.py 
+# # 1.0805372429895215

core/views/ui/drilldown.py

@@ -254,14 +254,11 @@ class DrilldownContextModal(APIView):
         nicks = None
         query = False
         # Create the query params from the POST arguments
-        mandatory = ["net", "channel", "num", "src", "index", "nick", "type"]
+        mandatory = ["net", "channel", "num", "src", "index", "nick", "type", "mtype"]
         invalid = [None, False, "—", "None"]
 
         query_params = {k: v for k, v in request.data.items() if v}
 
-        if query_params["index"] == "int":
-            mandatory.append("mtype")
-
         for key in query_params:
             if query_params[key] in invalid:
                 query_params[key] = None
@@ -314,13 +311,16 @@ class DrilldownContextModal(APIView):
         if query_params["src"] == "irc":
             if query_params["type"] in ["query", "notice", "msg", "highlight"]:
                 annotate = True
-
         results = query_results(
-            request, query_params, annotate=annotate, custom_query=search_query
+            request,
+            query_params,
+            annotate=annotate,
+            custom_query=search_query,
+            reverse=True,
+            dedup_fields=["net", "type", "msg"],
         )
         if "message" in results:
             return render(request, self.template_name, results)
-        results["object_list"] = results["object_list"][::-1]
 
         # Make the time nicer
         # for index, item in enumerate(results["object_list"]):
@@ -339,6 +339,7 @@ class DrilldownContextModal(APIView):
             "type": query_params["type"],
             "mtype": query_params["mtype"],
             "nick": query_params["nick"],
+            "params": query_params,
         }
         if request.user.is_superuser:
             if query:

core/views/ui/tables.py

@@ -66,5 +66,6 @@ class DrilldownTable(Table):
     server = Column()
     mtype = Column()
     tokens = Column()
+    hidden = Column()
     template_name = "ui/drilldown/table_results.html"
     paginate_by = settings.DRILLDOWN_RESULTS_PER_PAGE