diff --git a/core/lib/manticore.py b/core/lib/manticore.py
new file mode 100644
index 0000000..1132a95
--- /dev/null
+++ b/core/lib/manticore.py
@@ -0,0 +1,282 @@
+from re import search
+from django.conf import settings
+from core.lib.opensearch import annotate_results, filter_blacklisted, parse_results
+import manticoresearch
+from core.views.helpers import dedup_list
+
+
+def initialise_manticore():
+ """
+ Initialise the Manticore client
+ """
+ configuration = manticoresearch.Configuration(host="http://monolith-db-1:9308")
+ api_client = manticoresearch.ApiClient(configuration)
+ api_instance = manticoresearch.SearchApi(api_client)
+
+ return (api_client, api_instance)
+
+api_client, client = initialise_manticore()
+
+def construct_query(query, size, index, blank=False):
+ """
+ Accept some query parameters and construct an OpenSearch query.
+ """
+ if not size:
+ size = 5
+ query_base = {
+ "index": index,
+ "limit": size,
+ "query": {"bool": {"must": []}},
+ }
+ query_string = {
+ "query_string": query,
+ }
+ if not blank:
+ query_base["query"]["bool"]["must"].append(query_string)
+ return query_base
+
+def run_query(client, user, search_query):
+ response = client.search(search_query)
+ response = response.to_dict()
+ filter_blacklisted(user, response)
+ return response
+
+def query_results(
+ request,
+ query_params,
+ size=None,
+ annotate=True,
+ custom_query=False,
+ reverse=False,
+ dedup=False,
+ dedup_fields=None,
+ tags=None,
+):
+ query = None
+ message = None
+ message_class = None
+ add_bool = []
+ add_top = []
+ add_top_negative = []
+ sort = None
+ query_created = False
+ source = None
+
+ # Check size
+ if request.user.is_anonymous:
+ sizes = settings.MANTICORE_MAIN_SIZES_ANON
+ else:
+ sizes = settings.MANTICORE_MAIN_SIZES
+ if not size:
+ if "size" in query_params:
+ size = query_params["size"]
+ if size not in sizes:
+ message = "Size is not permitted"
+ message_class = "danger"
+ return {"message": message, "class": message_class}
+ size = int(size)
+ else:
+ size = 20
+
+ # Check index
+ if "index" in query_params:
+ index = query_params["index"]
+ if index == "main":
+ index = settings.MANTICORE_INDEX_MAIN
+ else:
+ if not request.user.has_perm(f"core.index_{index}"):
+ message = "Not permitted to search by this index"
+ message_class = "danger"
+ return {
+ "message": message,
+ "class": message_class,
+ }
+ if index == "meta":
+ index = settings.MANTICORE_INDEX_META
+ elif index == "int":
+ index = settings.MANTICORE_INDEX_INT
+ else:
+ message = "Index is not valid."
+ message_class = "danger"
+ return {
+ "message": message,
+ "class": message_class,
+ }
+ else:
+ index = settings.MANTICORE_INDEX_MAIN
+
+ # Create the search query
+ if "query" in query_params:
+ query = query_params["query"]
+ search_query = construct_query(query, size, index)
+ query_created = True
+
+ if tags:
+ # Get a blank search query
+ if not query_created:
+ search_query = construct_query(None, size, index, blank=True)
+ query_created = True
+ for tagname, tagvalue in tags.items():
+ add_bool.append({tagname: tagvalue})
+
+ required_any = ["query_full", "query", "tags"]
+ if not any([field in query_params.keys() for field in required_any]):
+ if not custom_query:
+ print("EMPTY QUERY")
+ message = "Empty query!"
+ message_class = "warning"
+ return {"message": message, "class": message_class}
+
+ # Check for a source
+ if "source" in query_params:
+ source = query_params["source"]
+
+ if source in settings.MANTICORE_SOURCES_RESTRICTED:
+ if not request.user.has_perm("core.restricted_sources"):
+ message = "Access denied"
+ message_class = "danger"
+ return {"message": message, "class": message_class}
+ elif source not in settings.MANTICORE_MAIN_SOURCES:
+ message = "Invalid source"
+ message_class = "danger"
+ return {"message": message, "class": message_class}
+
+ if source == "all":
+ source = None # the next block will populate it
+
+ if source:
+ sources = [source]
+ else:
+ sources = settings.MANTICORE_MAIN_SOURCES
+ if request.user.has_perm("core.restricted_sources"):
+ for source_iter in settings.MANTICORE_SOURCES_RESTRICTED:
+ sources.append(source_iter)
+
+ add_top_tmp = {"bool": {"should": []}}
+ for source_iter in sources:
+ add_top_tmp["bool"]["should"].append({"match_phrase": {"src": source_iter}})
+ add_top.append(add_top_tmp)
+
+
+ # Date/time range
+ if set({"from_date", "to_date", "from_time", "to_time"}).issubset(
+ query_params.keys()
+ ):
+ from_ts = f"{query_params['from_date']}T{query_params['from_time']}Z"
+ to_ts = f"{query_params['to_date']}T{query_params['to_time']}Z"
+ range_query = {
+ "range": {
+ "ts": {
+ "gt": from_ts,
+ "lt": to_ts,
+ }
+ }
+ }
+ add_top.append(range_query)
+
+ # Sorting
+ if "sorting" in query_params:
+ sorting = query_params["sorting"]
+ if sorting not in ("asc", "desc", "none"):
+ message = "Invalid sort"
+ message_class = "danger"
+ return {"message": message, "class": message_class}
+ if sorting in ("asc", "desc"):
+ sort = [
+ {
+ "ts": {
+ "order": sorting,
+ }
+ }
+ ]
+
+ # Sentiment handling
+ if "check_sentiment" in query_params:
+ if "sentiment_method" not in query_params:
+ message = "No sentiment method"
+ message_class = "danger"
+ return {"message": message, "class": message_class}
+ if "sentiment" in query_params:
+ sentiment = query_params["sentiment"]
+ try:
+ sentiment = float(sentiment)
+ except ValueError:
+ message = "Sentiment is not a float"
+ message_class = "danger"
+ return {"message": message, "class": message_class}
+ sentiment_method = query_params["sentiment_method"]
+ range_query_compare = {"range": {"sentiment": {}}}
+ range_query_precise = {
+ "match": {
+ "sentiment": None,
+ }
+ }
+ if sentiment_method == "below":
+ range_query_compare["range"]["sentiment"]["lt"] = sentiment
+ add_top.append(range_query_compare)
+ elif sentiment_method == "above":
+ range_query_compare["range"]["sentiment"]["gt"] = sentiment
+ add_top.append(range_query_compare)
+ elif sentiment_method == "exact":
+ range_query_precise["match"]["sentiment"] = sentiment
+ add_top.append(range_query_precise)
+ elif sentiment_method == "nonzero":
+ range_query_precise["match"]["sentiment"] = 0
+ add_top_negative.append(range_query_precise)
+
+ if add_bool:
+ # if "bool" not in search_query["query"]:
+ # search_query["query"]["bool"] = {}
+ # if "must" not in search_query["query"]["bool"]:
+ # search_query["query"]["bool"] = {"must": []}
+
+ for item in add_bool:
+ search_query["query"]["bool"]["must"].append({"match": item})
+
+ if add_top:
+ for item in add_top:
+ search_query["query"]["bool"]["must"].append(item)
+ if add_top_negative:
+ for item in add_top_negative:
+ if "must_not" in search_query["query"]["bool"]:
+ search_query["query"]["bool"]["must_not"].append(item)
+ else:
+ search_query["query"]["bool"]["must_not"] = [item]
+ if sort:
+ search_query["sort"] = sort
+
+
+ print("RUNNING QUERY", search_query)
+ results = run_query(
+ client,
+ request.user, # passed through run_main_query to filter_blacklisted
+ search_query,
+ )
+ if not results:
+ return False
+ #results = results.to_dict()
+ results_parsed = parse_results(results)
+ if annotate:
+ annotate_results(results_parsed)
+ if "dedup" in query_params:
+ if query_params["dedup"] == "on":
+ dedup = True
+ else:
+ dedup = False
+ else:
+ dedup = False
+
+ if reverse:
+ results_parsed = results_parsed[::-1]
+
+ if dedup:
+ if not dedup_fields:
+ dedup_fields = ["msg", "nick", "ident", "host", "net", "channel"]
+ results_parsed = dedup_list(results_parsed, dedup_fields)
+ context = {
+ "object_list": results_parsed,
+ "card": results["hits"]["total"],
+ "took": results["took"],
+ }
+ print("RETURN", context)
+ return context
\ No newline at end of file
diff --git a/core/lib/opensearch.py b/core/lib/opensearch.py
index a7568de..6a1c5a4 100644
--- a/core/lib/opensearch.py
+++ b/core/lib/opensearch.py
@@ -7,7 +7,7 @@ from opensearchpy.exceptions import NotFoundError, RequestError
from core.lib.threshold import annotate_num_chans, annotate_num_users, annotate_online
from core.views.helpers import dedup_list
-
+from datetime import datetime
# from json import dumps
# pp = lambda x: print(dumps(x, indent=2))
@@ -261,7 +261,10 @@ def parse_results(results):
del element["time"]
element["ts"] = ts
if "ts" in element:
- ts = element["ts"]
+ if isinstance(element["ts"], str):
+ ts = element["ts"]
+ else:
+ ts = datetime.utcfromtimestamp(element["ts"]).strftime('%Y-%m-%dT%H:%M:%S')
ts_spl = ts.split("T")
date = ts_spl[0]
time = ts_spl[1]
diff --git a/core/templates/widgets/table_results.html b/core/templates/widgets/table_results.html
index 1ae334f..f22a456 100644
--- a/core/templates/widgets/table_results.html
+++ b/core/templates/widgets/table_results.html
@@ -16,6 +16,7 @@
{% endblock %}
{% block panel_content %}
+ {% include 'partials/notify.html' %}
diff --git a/core/views/ui/drilldown.py b/core/views/ui/drilldown.py
index af6f407..8bf39cb 100644
--- a/core/views/ui/drilldown.py
+++ b/core/views/ui/drilldown.py
@@ -12,7 +12,8 @@ from rest_framework.parsers import FormParser
from rest_framework.views import APIView
from core.lib.context import construct_query
-from core.lib.opensearch import query_results
+#from core.lib.opensearch import query_results
+from core.lib.manticore import query_results
from core.lib.threshold import (
annotate_num_chans,
annotate_num_users,
@@ -175,6 +176,8 @@ def drilldown_search(request, return_context=False, template=None):
context["client_uri"] = url_params
context["params"] = query_params
if "message" in context:
+ if return_context:
+ return context
response = render(request, template_name, context)
if request.GET:
if request.htmx:
@@ -207,7 +210,6 @@ def drilldown_search(request, return_context=False, template=None):
context["class"] = "warning"
# unique = str(uuid.uuid4())[:8]
-
response = render(request, template_name, context)
if request.GET:
if request.htmx:
@@ -383,7 +385,6 @@ class DrilldownContextModal(APIView):
custom_query=search_query,
reverse=True,
dedup_fields=["net", "type", "msg"],
- lookup_hashes=False,
)
if "message" in results:
return render(request, self.template_name, results)
diff --git a/docker/requirements.dev.txt b/docker/requirements.dev.txt
index 4639f01..06710d2 100644
--- a/docker/requirements.dev.txt
+++ b/docker/requirements.dev.txt
@@ -13,3 +13,4 @@ cryptography
siphashc
redis
sortedcontainers
+manticoresearch
diff --git a/requirements.txt b/requirements.txt
index ff248c9..17a96c2 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -14,3 +14,4 @@ cryptography
siphashc
redis
sortedcontainers
+manticoresearch