Fix the graph rendering

core/db/__init__.py

@@ -1,6 +1,7 @@
 import random
 import string
 import time
+from datetime import datetime
 from math import floor, log10
 
 import orjson
@@ -10,6 +11,7 @@ from siphashc import siphash
 from core import r
 from core.db.processing import annotate_results
 from core.util import logs
+from core.views import helpers
 
 
 class StorageBackend(object):
@@ -71,6 +73,15 @@ class StorageBackend(object):
                     index = settings.INDEX_META
                 elif index == "internal":
                     index = settings.INDEX_INT
+                elif index == "restricted":
+                    if not user.has_perm("core.restricted_sources"):
+                        message = "Not permitted to search by this index"
+                        message_class = "danger"
+                        return {
+                            "message": message,
+                            "class": message_class,
+                        }
+                    index = settings.INDEX_RESTRICTED
                 else:
                     message = "Index is not valid."
                     message_class = "danger"
@@ -83,6 +94,7 @@ class StorageBackend(object):
         return index
 
     def parse_query(self, query_params, tags, size, index, custom_query, add_bool):
+        query_created = False
         if "query" in query_params:
             query = query_params["query"]
             search_query = self.construct_query(query, size, index)
@@ -90,15 +102,25 @@ class StorageBackend(object):
         else:
             if custom_query:
                 search_query = custom_query
+            else:
+                search_query = self.construct_query(None, size, index, blank=True)
 
         if tags:
             # Get a blank search query
             if not query_created:
                 search_query = self.construct_query(None, size, index, blank=True)
                 query_created = True
-            for tagname, tagvalue in tags.items():
+            for item in tags:
+                for tagname, tagvalue in item.items():
                     add_bool.append({tagname: tagvalue})
 
+        valid = self.check_valid_query(query_params, custom_query)
+        if isinstance(valid, dict):
+            return valid
+
+        return search_query
+
+    def check_valid_query(self, query_params, custom_query):
         required_any = ["query", "tags"]
         if not any([field in query_params.keys() for field in required_any]):
             if not custom_query:
@@ -106,8 +128,6 @@ class StorageBackend(object):
                 message_class = "warning"
                 return {"message": message, "class": message_class}
 
-        return search_query
-
     def parse_source(self, user, query_params):
         if "source" in query_params:
             source = query_params["source"]
@@ -133,11 +153,59 @@ class StorageBackend(object):
                 for source_iter in settings.SOURCES_RESTRICTED:
                     sources.append(source_iter)
 
+        if "all" in sources:
+            sources.remove("all")
+
         return sources
 
+    def parse_sort(self, query_params):
+        sort = None
+        if "sorting" in query_params:
+            sorting = query_params["sorting"]
+            if sorting not in ("asc", "desc", "none"):
+                message = "Invalid sort"
+                message_class = "danger"
+                return {"message": message, "class": message_class}
+            if sorting == "asc":
+                sort = "ascending"
+            elif sorting == "desc":
+                sort = "descending"
+        return sort
+
+    def parse_date_time(self, query_params):
+        if set({"from_date", "to_date", "from_time", "to_time"}).issubset(
+            query_params.keys()
+        ):
+            from_ts = f"{query_params['from_date']}T{query_params['from_time']}Z"
+            to_ts = f"{query_params['to_date']}T{query_params['to_time']}Z"
+            from_ts = datetime.strptime(from_ts, "%Y-%m-%dT%H:%MZ")
+            to_ts = datetime.strptime(to_ts, "%Y-%m-%dT%H:%MZ")
+
+            return (from_ts, to_ts)
+        return (None, None)
+
+    def parse_sentiment(self, query_params):
+        sentiment = None
+        if "check_sentiment" in query_params:
+            if "sentiment_method" not in query_params:
+                message = "No sentiment method"
+                message_class = "danger"
+                return {"message": message, "class": message_class}
+            if "sentiment" in query_params:
+                sentiment = query_params["sentiment"]
+                try:
+                    sentiment = float(sentiment)
+                except ValueError:
+                    message = "Sentiment is not a float"
+                    message_class = "danger"
+                    return {"message": message, "class": message_class}
+            sentiment_method = query_params["sentiment_method"]
+
+            return (sentiment_method, sentiment)
+
     def filter_blacklisted(self, user, response):
         """
-        Low level filter to take the raw OpenSearch response and remove
+        Low level filter to take the raw search response and remove
         objects from it we want to keep secret.
         Does not return, the object is mutated in place.
         """
@@ -197,10 +265,27 @@ class StorageBackend(object):
             cache_hit = r.get(f"query_cache.{user.id}.{hash}")
             if cache_hit:
                 response = orjson.loads(cache_hit)
-                response["cache"] = True
+                print("CACHE HIT", response)
-                return response
+
+                time_took = (time.process_time() - start) * 1000
+                # Round to 3 significant figures
+                time_took_rounded = round(
+                    time_took, 3 - int(floor(log10(abs(time_took)))) - 1
+                )
+                return {
+                    "object_list": response,
+                    "took": time_took_rounded,
+                    "cache": True,
+                }
         response = self.run_query(user, search_query)
-        if "error" in response and len(response.keys()) == 1:
+        if "error" in response:
+            if "errorMessage" in response:
+                context = {
+                    "message": response["errorMessage"],
+                    "class": "danger",
+                }
+                return context
+            else:
                 return response
         # response = response.to_dict()
         # print("RESP", response)
@@ -209,15 +294,15 @@ class StorageBackend(object):
                 return None
         self.filter_blacklisted(user, response)
 
-        # Write cache
-        if settings.CACHE:
-            to_write_cache = orjson.dumps(response)
-            r.set(f"query_cache.{user.id}.{hash}", to_write_cache)
-            r.expire(f"query_cache.{user.id}.{hash}", settings.CACHE_TIMEOUT)
-
         # Parse the response
         response_parsed = self.parse(response)
 
+        # Write cache
+        if settings.CACHE:
+            to_write_cache = orjson.dumps(response_parsed)
+            r.set(f"query_cache.{user.id}.{hash}", to_write_cache)
+            r.expire(f"query_cache.{user.id}.{hash}", settings.CACHE_TIMEOUT)
+
         time_took = (time.process_time() - start) * 1000
         # Round to 3 significant figures
         time_took_rounded = round(time_took, 3 - int(floor(log10(abs(time_took)))) - 1)
@@ -226,9 +311,15 @@ class StorageBackend(object):
     def query_results(self, **kwargs):
         raise NotImplementedError
 
-    def process_results(self, **kwargs):
+    def process_results(self, response, **kwargs):
         if kwargs.get("annotate"):
-            annotate_results(kwargs["results"])
+            annotate_results(response)
+        if kwargs.get("dedup"):
+            response = response[::-1]
+        if kwargs.get("dedup"):
+            if not kwargs.get("dedup_fields"):
+                dedup_fields = ["msg", "nick", "ident", "host", "net", "channel"]
+            response = helpers.dedup_list(response, dedup_fields)
 
     def parse(self, response):
         raise NotImplementedError

core/db/druid.py

@@ -1,17 +1,9 @@
 import logging
-import random
-import string
-import time
-from datetime import datetime
-from math import floor, log10
-from pprint import pprint
 
 import orjson
 import requests
 from django.conf import settings
-from siphashc import siphash
 
-from core import r
 from core.db import StorageBackend
 from core.db.processing import parse_druid
 from core.views import helpers
@@ -27,23 +19,76 @@ class DruidBackend(StorageBackend):
         # self.client = PyDruid("http://broker:8082", "druid/v2")
         pass  # we use requests
 
+    def construct_context_query(
+        self, index, net, channel, src, num, size, type=None, nicks=None
+    ):
+        search_query = self.construct_query(None, size, index, blank=True)
+        extra_must = []
+        extra_should = []
+        extra_should2 = []
+        if num:
+            extra_must.append({"num": num})
+        if net:
+            extra_must.append({"net": net})
+        if channel:
+            extra_must.append({"channel": channel})
+        if nicks:
+            for nick in nicks:
+                extra_should2.append({"nick": nick})
+        types = ["msg", "notice", "action", "kick", "topic", "mode"]
+
+        if index == "internal":
+            if channel == "*status" or type == "znc":
+                if {"channel": channel} in extra_must:
+                    extra_must.remove({"channel": channel})
+                extra_should2 = []
+                # Type is one of msg or notice
+                # extra_should.append({"match": {"mtype": "msg"}})
+                # extra_should.append({"match": {"mtype": "notice"}})
+                extra_should.append({"type": "znc"})
+                extra_should.append({"type": "self"})
+
+                extra_should2.append({"type": "znc"})
+                extra_should2.append({"nick": channel})
+            elif type == "auth":
+                if {"match": {"channel": channel}} in extra_must:
+                    extra_must.remove({"channel": channel})
+                extra_should2 = []
+                extra_should2.append({"nick": channel})
+                # extra_should2.append({"match": {"mtype": "msg"}})
+                # extra_should2.append({"match": {"mtype": "notice"}})
+
+                extra_should.append({"type": "query"})
+                extra_should2.append({"type": "self"})
+                extra_should.append({"nick": channel})
+            else:
+                for ctype in types:
+                    extra_should.append({"mtype": ctype})
+        else:
+            for ctype in types:
+                extra_should.append({"type": ctype})
+
+        if extra_must:
+            self.add_type("and", search_query, extra_must)
+
+        if extra_should:
+            self.add_type("or", search_query, extra_should)
+        if extra_should2:
+            self.add_type("or", search_query, extra_should2)
+        return search_query
+
     def construct_query(self, query, size, index, blank=False):
         search_query = {
             "limit": size,
             "queryType": "scan",
             "dataSource": index,
-            "filter": {
+            "intervals": ["1999-01-01/2999-01-01"],
-                "type": "and",
-                "fields": [
-
-                ],
-            },
-            # "resultFormat": "list",
-            # "columns":[],
-            "intervals": ["1000-01-01/3000-01-01"],
-            # "batchSize": 20480,
         }
 
+        base_filter = {
+            "type": "and",
+            "fields": [],
+        }
         to_add = {
             "type": "search",
             "dimension": "msg",
@@ -51,11 +96,12 @@ class DruidBackend(StorageBackend):
                 "type": "insensitive_contains",
                 "value": query,
             },
-                    },
+        }
 
         if blank:
             return search_query
         else:
+            search_query["filter"] = base_filter
             search_query["filter"]["fields"].append(to_add)
             return search_query
 
@@ -65,12 +111,15 @@ class DruidBackend(StorageBackend):
         return parsed
 
     def run_query(self, user, search_query):
+        ss = orjson.dumps(search_query, option=orjson.OPT_INDENT_2)
+        ss = ss.decode()
+        print(ss)
         response = requests.post("http://broker:8082/druid/v2", json=search_query)
         response = orjson.loads(response.text)
         print("RESPONSE LEN", len(response))
-        ss = orjson.dumps(list(response), option=orjson.OPT_INDENT_2)
+        # ss = orjson.dumps(response, option=orjson.OPT_INDENT_2)
-        ss = ss.decode()
+        # ss = ss.decode()
-        print(ss)
+        # print(ss)
         return response
 
     def filter_blacklisted(self, user, response):
@@ -89,12 +138,24 @@ class DruidBackend(StorageBackend):
         tags=None,
     ):
         add_bool = []
-        add_top = []
+        add_in = {}
-
 
         helpers.add_defaults(query_params)
 
-        # Check size
+        # Now, run the helpers for SIQTSRSS/ADR
+        # S - Size
+        # I - Index
+        # Q - Query
+        # T - Tags
+        # S - Source
+        # R - Ranges
+        # S - Sort
+        # S - Sentiment
+        # A - Annotate
+        # D - Dedup
+        # R - Reverse
+
+        # S - Size
         if request.user.is_anonymous:
             sizes = settings.MAIN_SIZES_ANON
         else:
@@ -104,37 +165,80 @@ class DruidBackend(StorageBackend):
             if isinstance(size, dict):
                 return size
 
-        # Check index
+        # I - Index
         index = self.parse_index(request.user, query_params)
         if isinstance(index, dict):
             return index
 
-        # Create the search query
+        # Q/T - Query/Tags
-        search_query = self.parse_query(query_params, tags, size, index, custom_query, add_bool)
+        search_query = self.parse_query(
-        if isinstance(search_query, dict):
+            query_params, tags, size, index, custom_query, add_bool
+        )
+        # Query should be a dict, so check if it contains message here
+        if "message" in search_query:
             return search_query
 
+        # S - Sources
         sources = self.parse_source(request.user, query_params)
-        # TODO
+        if isinstance(sources, dict):
-        add_top_tmp = {"bool": {"should": []}}
+            return sources
-        total_count = 0
+        total_count = len(sources)
-        for source_iter in sources:
+        total_sources = len(settings.MAIN_SOURCES) + len(settings.SOURCES_RESTRICTED)
-            add_top_tmp["bool"]["should"].append({"equals": {"src": source_iter}})
+        if total_count != total_sources:
-            total_count += 1
+            add_in["src"] = sources
-        total_sources = len(settings.MAIN_SOURCES) + len(
-            settings.SOURCES_RESTRICTED
-        )
-        if not total_count == total_sources:
-            add_top.append(add_top_tmp)
 
-        print("SIZE IS", size)
+        # R - Ranges
+        from_ts, to_ts = self.parse_date_time(query_params)
+        if from_ts:
+            addendum = f"{from_ts}/{to_ts}"
+            search_query["intervals"] = [addendum]
 
+        # S - Sort
+        sort = self.parse_sort(query_params)
+        if isinstance(sort, dict):
+            return sort
+        if sort:
+            search_query["order"] = sort
+
+        # S - Sentiment
+        sentiment_r = self.parse_sentiment(query_params)
+        if isinstance(sentiment_r, dict):
+            return sentiment_r
+        if sentiment_r:
+            sentiment_method, sentiment = sentiment_r
+            sentiment_query = {"type": "bound", "dimension": "sentiment"}
+            if sentiment_method == "below":
+                sentiment_query["upper"] = sentiment
+            elif sentiment_method == "above":
+                sentiment_query["lower"] = sentiment
+            elif sentiment_method == "exact":
+                sentiment_query["lower"] = sentiment
+                sentiment_query["upper"] = sentiment
+            elif sentiment_method == "nonzero":
+                sentiment_query["lower"] = -0.0001
+                sentiment_query["upper"] = 0.0001
+                sentiment_query["lowerStrict"] = True
+                sentiment_query["upperStrict"] = True
+            # add_bool.append(sentiment_query)
+            self.add_filter(search_query)
+            search_query["filter"]["fields"].append(sentiment_query)
+
+        # Add in the additional information we already populated
         if add_bool:
-            self.add_bool(search_query, add_bool)
+            self.add_type("and", search_query, add_bool)
+        if add_in:
+            self.add_in(search_query, add_in)
 
         response = self.query(request.user, search_query)
-        # print("RESP", response)
 
+        # A/D/R - Annotate/Dedup/Reverse
+        self.process_results(
+            response,
+            annotate=annotate,
+            dedup=dedup,
+            dedup_fields=dedup_fields,
+            reverse=reverse,
+        )
         # ss = orjson.dumps(list(response), option=orjson.OPT_INDENT_2)
         # ss = ss.decode()
         # print(ss)
@@ -143,11 +247,29 @@ class DruidBackend(StorageBackend):
         context = response
         return context
 
-    def add_bool(self, search_query, add_bool):
+    def add_filter(self, search_query):
-        if "filter" in search_query:
+        if "filter" not in search_query:
-            if "fields" in search_query["filter"]:
+            search_query["filter"] = {
-                search_query["filter"]["fields"].append({"bool": {"should": add_bool}})
+                "type": "and",
-            else:
+                "fields": [],
-                search_query["filter"]["fields"] = [{"bool": {"should": add_bool}}]
+            }
-        else:
+
-            search_query["filter"] = {"bool": {"should": add_bool}}
+    def add_in(self, search_query, add_in):
+        self.add_filter(search_query)
+        for key, value in add_in.items():
+            to_add = {"type": "in", "dimension": key, "values": value}
+            search_query["filter"]["fields"].append(to_add)
+
+    def add_type(self, gate, search_query, add_bool):
+        top_level_bool = {"type": gate, "fields": []}
+        self.add_filter(search_query)
+        for item in add_bool:
+            for key, value in item.items():
+                to_add = {"type": "selector", "dimension": key, "value": value}
+                top_level_bool["fields"].append(to_add)
+
+        search_query["filter"]["fields"].append(top_level_bool)
+
+    def check_valid_query(self, query_params, custom_query):
+        # We can do blank queries with this data source
+        pass

core/db/manticore.py

@@ -1,18 +1,12 @@
 import logging
-import random
-import string
-import time
 from datetime import datetime
-from math import floor, log10
 from pprint import pprint
 
-import orjson
 import requests
 from django.conf import settings
 
-from core import r
 from core.db import StorageBackend
-from core.db.processing import annotate_results, filter_blacklisted, parse_results
+from core.db.processing import annotate_results, parse_results
 from core.views import helpers
 
 logger = logging.getLogger(__name__)
@@ -120,7 +114,7 @@ class ManticoreBackend(StorageBackend):
         # Create the search query
         if "query" in query_params:
             query = query_params["query"]
-            search_query = construct_query(query, size, index)
+            search_query = self.construct_query(query, size, index)
             query_created = True
         else:
             if custom_query:
@@ -129,7 +123,7 @@ class ManticoreBackend(StorageBackend):
         if tags:
             # Get a blank search query
             if not query_created:
-                search_query = construct_query(None, size, index, blank=True)
+                search_query = self.construct_query(None, size, index, blank=True)
                 query_created = True
             for tagname, tagvalue in tags.items():
                 add_bool.append({tagname: tagvalue})
@@ -171,9 +165,7 @@ class ManticoreBackend(StorageBackend):
         for source_iter in sources:
             add_top_tmp["bool"]["should"].append({"equals": {"src": source_iter}})
             total_count += 1
-        total_sources = len(settings.MAIN_SOURCES) + len(
+        total_sources = len(settings.MAIN_SOURCES) + len(settings.SOURCES_RESTRICTED)
-            settings.SOURCES_RESTRICTED
-        )
         if not total_count == total_sources:
             add_top.append(add_top_tmp)
 
@@ -269,8 +261,8 @@ class ManticoreBackend(StorageBackend):
             search_query["sort"] = sort
 
         pprint(search_query)
-        results = run_query(
+        results = self.run_query(
-            client,
+            self.client,
             request.user,  # passed through run_main_query to filter_blacklisted
             search_query,
         )

core/db/opensearch.py

@@ -9,7 +9,7 @@ from core.db import StorageBackend
 
 # from json import dumps
 # pp = lambda x: print(dumps(x, indent=2))
-from core.db.processing import annotate_results, filter_blacklisted, parse_results
+from core.db.processing import annotate_results, parse_results
 from core.views.helpers import dedup_list
 
 

core/db/processing.py

@@ -1,7 +1,5 @@
 from datetime import datetime
 
-from django.conf import settings
-
 from core.lib.threshold import annotate_num_chans, annotate_num_users, annotate_online
 
 

core/lib/manage/threshold.py

@@ -1,5 +1,3 @@
-from django.conf import settings
-
 from core.lib.threshold import threshold_request
 
 

core/static/js/column-shifter.js

@@ -43,6 +43,7 @@ $(document).ready(function(){
                     "guild_member_count": "off",
                     "bot": "off",
                     "msg_id": "off",
+                    "user": "off",
                     "net_id": "off",
                     "user_id": "off",
                     "nick_id": "off",
@@ -63,6 +64,12 @@ $(document).ready(function(){
                     "file_md5": "off",
                     "file_ext": "off",
                     "file_size": "off",
+                    "lang_code": "off",
+                    //"lang_name": "off",
+                    "words_noun": "off",
+                    "words_adj": "off",
+                    "words_verb": "off",
+                    "words_adv": "off"
                 },
             };
         } else {

core/templates/ui/drilldown/search_partial.html

@@ -366,6 +366,12 @@
                   <option value="meta">Meta</option>
                 {% endif %}
 
+                {% if params.index == 'restricted' %}
+                  <option selected value="restricted">Restricted</option>
+                {% else %}
+                  <option value="restricted">Restricted</option>
+                {% endif %}
+
               </select>
               <span class="icon is-small is-left">
                 <i class="fas fa-magnifying-glass"></i>

core/templates/ui/drilldown/table_results_partial.html

@@ -364,6 +364,26 @@
                                   </span>
                                 {% endif %}
                               </td>
+                            {% elif column.name|slice:":6" == "words_" %}
+                              <td class="{{ column.name }}">
+                                {% if cell.0.1|length == 0 %}
+                                  <a
+                                    class="tag is-info"
+                                    onclick="populateSearch('{{ column.name }}', '{{ cell }}')">
+                                    {{ cell }}
+                                  </a>
+                                {% else %}
+                                  <div class="tags">
+                                    {% for word in cell %}
+                                      <a
+                                        class="tag is-info"
+                                        onclick="populateSearch('{{ column.name }}', '{{ word }}')">
+                                        {{ word }}
+                                      </a>
+                                    {% endfor %}
+                                  </div>
+                                {% endif %}
+                              </td>
                             {% else %}
                               <td class="{{ column.name }}">
                                 <a

core/templates/widgets/table_results.html

@@ -24,7 +24,7 @@
     </span>
   {% endif %}
 
-  fetched {{ table.data|length }} of {{ card }} hits in {{ took }}ms
+  fetched {{ table.data|length }} hits in {{ took }}ms
 
   {% if exemption is not None %}
     <span class="icon has-tooltip-bottom" data-tooltip="God mode">

core/views/ui/drilldown.py

@@ -1,7 +1,7 @@
 import urllib
 import uuid
 
-import ujson
+import orjson
 from django.conf import settings
 from django.http import HttpResponse, JsonResponse
 from django.shortcuts import render
@@ -12,7 +12,6 @@ from rest_framework.parsers import FormParser
 from rest_framework.views import APIView
 
 from core.db.storage import db
-from core.lib.context import construct_query
 from core.lib.threshold import (
     annotate_num_chans,
     annotate_num_users,
@@ -61,14 +60,14 @@ def parse_tags(tags_pre):
     """
     Parse the tags from the variable tags_pre.
     """
-    tags = {}
+    tags = []
     tags_spl = tags_pre.split(",")
     if tags_spl:
         for tag in tags_spl:
             tag = tag.split(": ")
             if len(tag) == 2:
                 key, val = tag
-                tags[key] = val
+                tags.append({key: val})
     return tags
 
 
@@ -95,7 +94,7 @@ def make_graph(results):
                 "date": date,
             }
         )
-    return ujson.dumps(graph)
+    return orjson.dumps(graph).decode("utf-8")
 
 
 def drilldown_search(request, return_context=False, template=None):
@@ -361,9 +360,14 @@ class DrilldownContextModal(APIView):
             if query_params["type"] not in ["znc", "auth"]:
                 annotate = True
         # Create the query with the context helper
+        if "num" in query_params:
+            if query_params["num"]:
                 if query_params["num"].isdigit():
                     query_params["num"] = int(query_params["num"])
-        search_query = construct_query(
+                else:
+                    return {"message": "Invalid num value", "class": "danger"}
+
+        search_query = db.construct_context_query(
             query_params["index"],
             query_params["net"],
             query_params["channel"],