Add manticore client
This commit is contained in:
parent
aeaf7bba5d
commit
017a05880b
GPG Key ID:
5ACFCEED46C0904F
|
|
@ -0,0 +1,282 @@
|
|||
from re import search
|
||||
from django.conf import settings
|
||||
from core.lib.opensearch import annotate_results, filter_blacklisted, parse_results
|
||||
import manticoresearch
|
||||
from core.views.helpers import dedup_list
|
||||
|
||||
|
||||
def initialise_manticore():
|
||||
"""
|
||||
Initialise the Manticore client
|
||||
"""
|
||||
configuration = manticoresearch.Configuration(host="http://monolith-db-1:9308")
|
||||
api_client = manticoresearch.ApiClient(configuration)
|
||||
api_instance = manticoresearch.SearchApi(api_client)
|
||||
|
||||
return (api_client, api_instance)
|
||||
|
||||
api_client, client = initialise_manticore()
|
||||
|
||||
def construct_query(query, size, index, blank=False):
|
||||
"""
|
||||
Accept some query parameters and construct an OpenSearch query.
|
||||
"""
|
||||
if not size:
|
||||
size = 5
|
||||
query_base = {
|
||||
"index": index,
|
||||
"limit": size,
|
||||
"query": {"bool": {"must": []}},
|
||||
}
|
||||
query_string = {
|
||||
"query_string": query,
|
||||
}
|
||||
if not blank:
|
||||
query_base["query"]["bool"]["must"].append(query_string)
|
||||
return query_base
|
||||
|
||||
def run_query(client, user, search_query):
|
||||
response = client.search(search_query)
|
||||
response = response.to_dict()
|
||||
filter_blacklisted(user, response)
|
||||
return response
|
||||
|
||||
def query_results(
|
||||
request,
|
||||
query_params,
|
||||
size=None,
|
||||
annotate=True,
|
||||
custom_query=False,
|
||||
reverse=False,
|
||||
dedup=False,
|
||||
dedup_fields=None,
|
||||
tags=None,
|
||||
):
|
||||
query = None
|
||||
message = None
|
||||
message_class = None
|
||||
add_bool = []
|
||||
add_top = []
|
||||
add_top_negative = []
|
||||
sort = None
|
||||
query_created = False
|
||||
source = None
|
||||
|
||||
# Check size
|
||||
if request.user.is_anonymous:
|
||||
sizes = settings.MANTICORE_MAIN_SIZES_ANON
|
||||
else:
|
||||
sizes = settings.MANTICORE_MAIN_SIZES
|
||||
if not size:
|
||||
if "size" in query_params:
|
||||
size = query_params["size"]
|
||||
if size not in sizes:
|
||||
message = "Size is not permitted"
|
||||
message_class = "danger"
|
||||
return {"message": message, "class": message_class}
|
||||
size = int(size)
|
||||
else:
|
||||
size = 20
|
||||
|
||||
# Check index
|
||||
if "index" in query_params:
|
||||
index = query_params["index"]
|
||||
if index == "main":
|
||||
index = settings.MANTICORE_INDEX_MAIN
|
||||
else:
|
||||
if not request.user.has_perm(f"core.index_{index}"):
|
||||
message = "Not permitted to search by this index"
|
||||
message_class = "danger"
|
||||
return {
|
||||
"message": message,
|
||||
"class": message_class,
|
||||
}
|
||||
if index == "meta":
|
||||
index = settings.MANTICORE_INDEX_META
|
||||
elif index == "int":
|
||||
index = settings.MANTICORE_INDEX_INT
|
||||
else:
|
||||
message = "Index is not valid."
|
||||
message_class = "danger"
|
||||
return {
|
||||
"message": message,
|
||||
"class": message_class,
|
||||
}
|
||||
else:
|
||||
index = settings.MANTICORE_INDEX_MAIN
|
||||
|
||||
# Create the search query
|
||||
if "query" in query_params:
|
||||
query = query_params["query"]
|
||||
search_query = construct_query(query, size, index)
|
||||
query_created = True
|
||||
|
||||
if tags:
|
||||
# Get a blank search query
|
||||
if not query_created:
|
||||
search_query = construct_query(None, size, index, blank=True)
|
||||
query_created = True
|
||||
for tagname, tagvalue in tags.items():
|
||||
add_bool.append({tagname: tagvalue})
|
||||
|
||||
required_any = ["query_full", "query", "tags"]
|
||||
if not any([field in query_params.keys() for field in required_any]):
|
||||
if not custom_query:
|
||||
print("EMPTY QUERY")
|
||||
message = "Empty query!"
|
||||
message_class = "warning"
|
||||
return {"message": message, "class": message_class}
|
||||
|
||||
# Check for a source
|
||||
if "source" in query_params:
|
||||
source = query_params["source"]
|
||||
|
||||
if source in settings.MANTICORE_SOURCES_RESTRICTED:
|
||||
if not request.user.has_perm("core.restricted_sources"):
|
||||
message = "Access denied"
|
||||
message_class = "danger"
|
||||
return {"message": message, "class": message_class}
|
||||
elif source not in settings.MANTICORE_MAIN_SOURCES:
|
||||
message = "Invalid source"
|
||||
message_class = "danger"
|
||||
return {"message": message, "class": message_class}
|
||||
|
||||
if source == "all":
|
||||
source = None # the next block will populate it
|
||||
|
||||
if source:
|
||||
sources = [source]
|
||||
else:
|
||||
sources = settings.MANTICORE_MAIN_SOURCES
|
||||
if request.user.has_perm("core.restricted_sources"):
|
||||
for source_iter in settings.MANTICORE_SOURCES_RESTRICTED:
|
||||
sources.append(source_iter)
|
||||
|
||||
add_top_tmp = {"bool": {"should": []}}
|
||||
for source_iter in sources:
|
||||
add_top_tmp["bool"]["should"].append({"match_phrase": {"src": source_iter}})
|
||||
add_top.append(add_top_tmp)
|
||||
|
||||
|
||||
# Date/time range
|
||||
if set({"from_date", "to_date", "from_time", "to_time"}).issubset(
|
||||
query_params.keys()
|
||||
):
|
||||
from_ts = f"{query_params['from_date']}T{query_params['from_time']}Z"
|
||||
to_ts = f"{query_params['to_date']}T{query_params['to_time']}Z"
|
||||
range_query = {
|
||||
"range": {
|
||||
"ts": {
|
||||
"gt": from_ts,
|
||||
"lt": to_ts,
|
||||
}
|
||||
}
|
||||
}
|
||||
add_top.append(range_query)
|
||||
|
||||
# Sorting
|
||||
if "sorting" in query_params:
|
||||
sorting = query_params["sorting"]
|
||||
if sorting not in ("asc", "desc", "none"):
|
||||
message = "Invalid sort"
|
||||
message_class = "danger"
|
||||
return {"message": message, "class": message_class}
|
||||
if sorting in ("asc", "desc"):
|
||||
sort = [
|
||||
{
|
||||
"ts": {
|
||||
"order": sorting,
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
# Sentiment handling
|
||||
if "check_sentiment" in query_params:
|
||||
if "sentiment_method" not in query_params:
|
||||
message = "No sentiment method"
|
||||
message_class = "danger"
|
||||
return {"message": message, "class": message_class}
|
||||
if "sentiment" in query_params:
|
||||
sentiment = query_params["sentiment"]
|
||||
try:
|
||||
sentiment = float(sentiment)
|
||||
except ValueError:
|
||||
message = "Sentiment is not a float"
|
||||
message_class = "danger"
|
||||
return {"message": message, "class": message_class}
|
||||
sentiment_method = query_params["sentiment_method"]
|
||||
range_query_compare = {"range": {"sentiment": {}}}
|
||||
range_query_precise = {
|
||||
"match": {
|
||||
"sentiment": None,
|
||||
}
|
||||
}
|
||||
if sentiment_method == "below":
|
||||
range_query_compare["range"]["sentiment"]["lt"] = sentiment
|
||||
add_top.append(range_query_compare)
|
||||
elif sentiment_method == "above":
|
||||
range_query_compare["range"]["sentiment"]["gt"] = sentiment
|
||||
add_top.append(range_query_compare)
|
||||
elif sentiment_method == "exact":
|
||||
range_query_precise["match"]["sentiment"] = sentiment
|
||||
add_top.append(range_query_precise)
|
||||
elif sentiment_method == "nonzero":
|
||||
range_query_precise["match"]["sentiment"] = 0
|
||||
add_top_negative.append(range_query_precise)
|
||||
|
||||
if add_bool:
|
||||
# if "bool" not in search_query["query"]:
|
||||
# search_query["query"]["bool"] = {}
|
||||
# if "must" not in search_query["query"]["bool"]:
|
||||
# search_query["query"]["bool"] = {"must": []}
|
||||
|
||||
for item in add_bool:
|
||||
search_query["query"]["bool"]["must"].append({"match": item})
|
||||
|
||||
if add_top:
|
||||
for item in add_top:
|
||||
search_query["query"]["bool"]["must"].append(item)
|
||||
if add_top_negative:
|
||||
for item in add_top_negative:
|
||||
if "must_not" in search_query["query"]["bool"]:
|
||||
search_query["query"]["bool"]["must_not"].append(item)
|
||||
else:
|
||||
search_query["query"]["bool"]["must_not"] = [item]
|
||||
if sort:
|
||||
search_query["sort"] = sort
|
||||
|
||||
|
||||
print("RUNNING QUERY", search_query)
|
||||
results = run_query(
|
||||
client,
|
||||
request.user, # passed through run_main_query to filter_blacklisted
|
||||
search_query,
|
||||
)
|
||||
if not results:
|
||||
return False
|
||||
#results = results.to_dict()
|
||||
results_parsed = parse_results(results)
|
||||
if annotate:
|
||||
annotate_results(results_parsed)
|
||||
if "dedup" in query_params:
|
||||
if query_params["dedup"] == "on":
|
||||
dedup = True
|
||||
else:
|
||||
dedup = False
|
||||
else:
|
||||
dedup = False
|
||||
|
||||
if reverse:
|
||||
results_parsed = results_parsed[::-1]
|
||||
|
||||
if dedup:
|
||||
if not dedup_fields:
|
||||
dedup_fields = ["msg", "nick", "ident", "host", "net", "channel"]
|
||||
results_parsed = dedup_list(results_parsed, dedup_fields)
|
||||
context = {
|
||||
"object_list": results_parsed,
|
||||
"card": results["hits"]["total"],
|
||||
"took": results["took"],
|
||||
}
|
||||
print("RETURN", context)
|
||||
return context
|
||||
|
|
@ -7,7 +7,7 @@ from opensearchpy.exceptions import NotFoundError, RequestError
|
|||
|
||||
from core.lib.threshold import annotate_num_chans, annotate_num_users, annotate_online
|
||||
from core.views.helpers import dedup_list
|
||||
|
||||
from datetime import datetime
|
||||
# from json import dumps
|
||||
# pp = lambda x: print(dumps(x, indent=2))
|
||||
|
||||
|
|
@ -261,7 +261,10 @@ def parse_results(results):
|
|||
del element["time"]
|
||||
element["ts"] = ts
|
||||
if "ts" in element:
|
||||
ts = element["ts"]
|
||||
if isinstance(element["ts"], str):
|
||||
ts = element["ts"]
|
||||
else:
|
||||
ts = datetime.utcfromtimestamp(element["ts"]).strftime('%Y-%m-%dT%H:%M:%S')
|
||||
ts_spl = ts.split("T")
|
||||
date = ts_spl[0]
|
||||
time = ts_spl[1]
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@
|
|||
{% endblock %}
|
||||
|
||||
{% block panel_content %}
|
||||
{% include 'partials/notify.html' %}
|
||||
<script src="{% static 'js/column-shifter.js' %}"></script>
|
||||
<span class="icon has-tooltip-bottom" data-tooltip="{{ card }} hits total">
|
||||
<i class="fa-solid fa-chart-mixed"></i>
|
||||
|
|
|
|||
|
|
@ -12,7 +12,8 @@ from rest_framework.parsers import FormParser
|
|||
from rest_framework.views import APIView
|
||||
|
||||
from core.lib.context import construct_query
|
||||
from core.lib.opensearch import query_results
|
||||
#from core.lib.opensearch import query_results
|
||||
from core.lib.manticore import query_results
|
||||
from core.lib.threshold import (
|
||||
annotate_num_chans,
|
||||
annotate_num_users,
|
||||
|
|
@ -175,6 +176,8 @@ def drilldown_search(request, return_context=False, template=None):
|
|||
context["client_uri"] = url_params
|
||||
context["params"] = query_params
|
||||
if "message" in context:
|
||||
if return_context:
|
||||
return context
|
||||
response = render(request, template_name, context)
|
||||
if request.GET:
|
||||
if request.htmx:
|
||||
|
|
@ -207,7 +210,6 @@ def drilldown_search(request, return_context=False, template=None):
|
|||
context["class"] = "warning"
|
||||
|
||||
# unique = str(uuid.uuid4())[:8]
|
||||
|
||||
response = render(request, template_name, context)
|
||||
if request.GET:
|
||||
if request.htmx:
|
||||
|
|
@ -383,7 +385,6 @@ class DrilldownContextModal(APIView):
|
|||
custom_query=search_query,
|
||||
reverse=True,
|
||||
dedup_fields=["net", "type", "msg"],
|
||||
lookup_hashes=False,
|
||||
)
|
||||
if "message" in results:
|
||||
return render(request, self.template_name, results)
|
||||
|
|
|
|||
|
|
@ -13,3 +13,4 @@ cryptography
|
|||
siphashc
|
||||
redis
|
||||
sortedcontainers
|
||||
manticoresearch
|
||||
|
|
|
|||
|
|
@ -14,3 +14,4 @@ cryptography
|
|||
siphashc
|
||||
redis
|
||||
sortedcontainers
|
||||
manticoresearch
|
||||
|
|
|
|||
Loading…
Reference in New Issue