Add manticore client

core/lib/manticore.py

@@ -0,0 +1,282 @@
+from re import search
+from django.conf import settings
+from core.lib.opensearch import annotate_results, filter_blacklisted, parse_results
+import manticoresearch
+from core.views.helpers import dedup_list
+
+
+def initialise_manticore():
+    """
+    Initialise the Manticore client
+    """
+    configuration = manticoresearch.Configuration(host="http://monolith-db-1:9308")
+    api_client = manticoresearch.ApiClient(configuration)
+    api_instance = manticoresearch.SearchApi(api_client)
+
+    return (api_client, api_instance)
+
+api_client, client = initialise_manticore()
+
+def construct_query(query, size, index, blank=False):
+    """
+    Accept some query parameters and construct an OpenSearch query.
+    """
+    if not size:
+        size = 5
+    query_base = {
+        "index": index,
+        "limit": size,
+        "query": {"bool": {"must": []}},
+    }
+    query_string = {
+        "query_string": query,
+    }
+    if not blank:
+        query_base["query"]["bool"]["must"].append(query_string)
+    return query_base
+
+def run_query(client, user, search_query):
+    response = client.search(search_query)
+    response = response.to_dict()
+    filter_blacklisted(user, response)
+    return response
+
+def query_results(
+    request,
+    query_params,
+    size=None,
+    annotate=True,
+    custom_query=False,
+    reverse=False,
+    dedup=False,
+    dedup_fields=None,
+    tags=None,
+):
+    query = None
+    message = None
+    message_class = None
+    add_bool = []
+    add_top = []
+    add_top_negative = []
+    sort = None
+    query_created = False
+    source = None
+
+    # Check size
+    if request.user.is_anonymous:
+        sizes = settings.MANTICORE_MAIN_SIZES_ANON
+    else:
+        sizes = settings.MANTICORE_MAIN_SIZES
+    if not size:
+        if "size" in query_params:
+            size = query_params["size"]
+            if size not in sizes:
+                message = "Size is not permitted"
+                message_class = "danger"
+                return {"message": message, "class": message_class}
+            size = int(size)
+        else:
+            size = 20
+
+    # Check index
+    if "index" in query_params:
+        index = query_params["index"]
+        if index == "main":
+            index = settings.MANTICORE_INDEX_MAIN
+        else:
+            if not request.user.has_perm(f"core.index_{index}"):
+                message = "Not permitted to search by this index"
+                message_class = "danger"
+                return {
+                    "message": message,
+                    "class": message_class,
+                }
+            if index == "meta":
+                index = settings.MANTICORE_INDEX_META
+            elif index == "int":
+                index = settings.MANTICORE_INDEX_INT
+            else:
+                message = "Index is not valid."
+                message_class = "danger"
+                return {
+                    "message": message,
+                    "class": message_class,
+                }
+    else:
+        index = settings.MANTICORE_INDEX_MAIN
+
+    # Create the search query
+    if "query" in query_params:
+        query = query_params["query"]
+        search_query = construct_query(query, size, index)
+        query_created = True
+
+    if tags:
+        # Get a blank search query
+        if not query_created:
+            search_query = construct_query(None, size, index, blank=True)
+            query_created = True
+        for tagname, tagvalue in tags.items():
+            add_bool.append({tagname: tagvalue})
+
+    required_any = ["query_full", "query", "tags"]
+    if not any([field in query_params.keys() for field in required_any]):
+        if not custom_query:
+            print("EMPTY QUERY")
+            message = "Empty query!"
+            message_class = "warning"
+            return {"message": message, "class": message_class}
+
+    # Check for a source
+    if "source" in query_params:
+        source = query_params["source"]
+
+        if source in settings.MANTICORE_SOURCES_RESTRICTED:
+            if not request.user.has_perm("core.restricted_sources"):
+                message = "Access denied"
+                message_class = "danger"
+                return {"message": message, "class": message_class}
+        elif source not in settings.MANTICORE_MAIN_SOURCES:
+            message = "Invalid source"
+            message_class = "danger"
+            return {"message": message, "class": message_class}
+
+        if source == "all":
+            source = None  # the next block will populate it
+
+    if source:
+        sources = [source]
+    else:
+        sources = settings.MANTICORE_MAIN_SOURCES
+        if request.user.has_perm("core.restricted_sources"):
+            for source_iter in settings.MANTICORE_SOURCES_RESTRICTED:
+                sources.append(source_iter)
+
+    add_top_tmp = {"bool": {"should": []}}
+    for source_iter in sources:
+        add_top_tmp["bool"]["should"].append({"match_phrase": {"src": source_iter}})
+    add_top.append(add_top_tmp)
+
+
+    # Date/time range
+    if set({"from_date", "to_date", "from_time", "to_time"}).issubset(
+        query_params.keys()
+    ):
+        from_ts = f"{query_params['from_date']}T{query_params['from_time']}Z"
+        to_ts = f"{query_params['to_date']}T{query_params['to_time']}Z"
+        range_query = {
+            "range": {
+                "ts": {
+                    "gt": from_ts,
+                    "lt": to_ts,
+                }
+            }
+        }
+        add_top.append(range_query)
+
+    # Sorting
+    if "sorting" in query_params:
+        sorting = query_params["sorting"]
+        if sorting not in ("asc", "desc", "none"):
+            message = "Invalid sort"
+            message_class = "danger"
+            return {"message": message, "class": message_class}
+        if sorting in ("asc", "desc"):
+            sort = [
+                {
+                    "ts": {
+                        "order": sorting,
+                    }
+                }
+            ]
+
+    # Sentiment handling
+    if "check_sentiment" in query_params:
+        if "sentiment_method" not in query_params:
+            message = "No sentiment method"
+            message_class = "danger"
+            return {"message": message, "class": message_class}
+        if "sentiment" in query_params:
+            sentiment = query_params["sentiment"]
+            try:
+                sentiment = float(sentiment)
+            except ValueError:
+                message = "Sentiment is not a float"
+                message_class = "danger"
+                return {"message": message, "class": message_class}
+        sentiment_method = query_params["sentiment_method"]
+        range_query_compare = {"range": {"sentiment": {}}}
+        range_query_precise = {
+            "match": {
+                "sentiment": None,
+            }
+        }
+        if sentiment_method == "below":
+            range_query_compare["range"]["sentiment"]["lt"] = sentiment
+            add_top.append(range_query_compare)
+        elif sentiment_method == "above":
+            range_query_compare["range"]["sentiment"]["gt"] = sentiment
+            add_top.append(range_query_compare)
+        elif sentiment_method == "exact":
+            range_query_precise["match"]["sentiment"] = sentiment
+            add_top.append(range_query_precise)
+        elif sentiment_method == "nonzero":
+            range_query_precise["match"]["sentiment"] = 0
+            add_top_negative.append(range_query_precise)
+
+    if add_bool:
+        # if "bool" not in search_query["query"]:
+        #     search_query["query"]["bool"] = {}
+        # if "must" not in search_query["query"]["bool"]:
+        #     search_query["query"]["bool"] = {"must": []}
+
+        for item in add_bool:
+            search_query["query"]["bool"]["must"].append({"match": item})
+
+    if add_top:
+        for item in add_top:
+            search_query["query"]["bool"]["must"].append(item)
+    if add_top_negative:
+        for item in add_top_negative:
+            if "must_not" in search_query["query"]["bool"]:
+                search_query["query"]["bool"]["must_not"].append(item)
+            else:
+                search_query["query"]["bool"]["must_not"] = [item]
+    if sort:
+        search_query["sort"] = sort
+
+
+    print("RUNNING QUERY", search_query)
+    results = run_query(
+        client,
+        request.user,  # passed through run_main_query to filter_blacklisted
+        search_query,
+    )
+    if not results:
+        return False
+    #results = results.to_dict()
+    results_parsed = parse_results(results)
+    if annotate:
+        annotate_results(results_parsed)
+    if "dedup" in query_params:
+        if query_params["dedup"] == "on":
+            dedup = True
+        else:
+            dedup = False
+    else:
+        dedup = False
+
+    if reverse:
+        results_parsed = results_parsed[::-1]
+
+    if dedup:
+        if not dedup_fields:
+            dedup_fields = ["msg", "nick", "ident", "host", "net", "channel"]
+        results_parsed = dedup_list(results_parsed, dedup_fields)
+    context = {
+        "object_list": results_parsed,
+        "card": results["hits"]["total"],
+        "took": results["took"],
+    }
+    print("RETURN", context)
+    return context

core/lib/opensearch.py

@@ -7,7 +7,7 @@ from opensearchpy.exceptions import NotFoundError, RequestError
 
 from core.lib.threshold import annotate_num_chans, annotate_num_users, annotate_online
 from core.views.helpers import dedup_list
-
+from datetime import datetime
 # from json import dumps
 # pp = lambda x: print(dumps(x, indent=2))
 
@@ -261,7 +261,10 @@ def parse_results(results):
                         del element["time"]
                         element["ts"] = ts
                 if "ts" in element:
+                    if isinstance(element["ts"], str):
                         ts = element["ts"]
+                    else:
+                        ts = datetime.utcfromtimestamp(element["ts"]).strftime('%Y-%m-%dT%H:%M:%S')
                     ts_spl = ts.split("T")
                     date = ts_spl[0]
                     time = ts_spl[1]

core/templates/widgets/table_results.html

@@ -16,6 +16,7 @@
 {% endblock %}
 
 {% block panel_content %}
+  {% include 'partials/notify.html' %}
   <script src="{% static 'js/column-shifter.js' %}"></script>
   <span class="icon has-tooltip-bottom" data-tooltip="{{ card }} hits total">
     <i class="fa-solid fa-chart-mixed"></i>

core/views/ui/drilldown.py

@@ -12,7 +12,8 @@ from rest_framework.parsers import FormParser
 from rest_framework.views import APIView
 
 from core.lib.context import construct_query
-from core.lib.opensearch import query_results
+#from core.lib.opensearch import query_results
+from core.lib.manticore import query_results
 from core.lib.threshold import (
     annotate_num_chans,
     annotate_num_users,
@@ -175,6 +176,8 @@ def drilldown_search(request, return_context=False, template=None):
     context["client_uri"] = url_params
     context["params"] = query_params
     if "message" in context:
+        if return_context:
+            return context
         response = render(request, template_name, context)
         if request.GET:
             if request.htmx:
@@ -207,7 +210,6 @@ def drilldown_search(request, return_context=False, template=None):
         context["class"] = "warning"
 
     # unique = str(uuid.uuid4())[:8]
-
     response = render(request, template_name, context)
     if request.GET:
         if request.htmx:
@@ -383,7 +385,6 @@ class DrilldownContextModal(APIView):
             custom_query=search_query,
             reverse=True,
             dedup_fields=["net", "type", "msg"],
-            lookup_hashes=False,
         )
         if "message" in results:
             return render(request, self.template_name, results)

docker/requirements.dev.txt

@@ -13,3 +13,4 @@ cryptography
 siphashc
 redis
 sortedcontainers
+manticoresearch

requirements.txt

@@ -14,3 +14,4 @@ cryptography
 siphashc
 redis
 sortedcontainers
+manticoresearch