From 703f36751ddc651efa50e60c8dec965238baa3c5 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Tue, 9 Aug 2022 07:20:30 +0100 Subject: [PATCH] Implement scrollback modal --- core/lib/__init__.py | 0 core/lib/context.py | 31 +++ core/lib/opensearch.py | 246 ++++++++++-------- core/static/js/column-shifter.js | 4 +- core/static/modal.js | 18 +- core/templates/modals/context.html | 85 +++++- core/templates/ui/drilldown/drilldown.html | 16 ++ .../templates/ui/drilldown/table_results.html | 4 +- .../ui/drilldown/table_results_partial.html | 6 +- core/views/ui/drilldown.py | 55 +++- 10 files changed, 338 insertions(+), 127 deletions(-) create mode 100644 core/lib/__init__.py create mode 100644 core/lib/context.py diff --git a/core/lib/__init__.py b/core/lib/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/core/lib/context.py b/core/lib/context.py new file mode 100644 index 0000000..cf474b6 --- /dev/null +++ b/core/lib/context.py @@ -0,0 +1,31 @@ +from core.lib.opensearch import client, run_main_query +from django.conf import settings + +def construct_query(net, channel, src, num, size): + # Get the initial query + extra_must = [] + if num: + extra_must.append({"match": {"num": num}}) + types = ["msg", "notice", "action", "kick", "topic", "mode"] + query_should = [{"match": {"type": x}} for x in types] + query = { + "size": size, + "query": { + "bool": { + "must": [ + {"match": {"net": net}}, + {"match": {"channel": channel}}, + {"match": {"src": src}}, + { + "bool": { + "should": query_should, + } + }, + *extra_must + ] + } + }, + "fields": ["nick", "ident", "host", "channel", "ts", "msg", "type", "net", "src"], + "_source": False, + } + return query \ No newline at end of file diff --git a/core/lib/opensearch.py b/core/lib/opensearch.py index 65726c4..bff8449 100644 --- a/core/lib/opensearch.py +++ b/core/lib/opensearch.py @@ -97,8 +97,14 @@ def filter_blacklisted(user, response): # For every blacklisted type for blacklisted_type in settings.OPENSEARCH_BLACKLISTED.keys(): # Check this field we are matching exists - if blacklisted_type in item["_source"].keys(): - content = item["_source"][blacklisted_type] + if "_source" in item.keys(): + data_index = "_source" + elif "fields" in item.keys(): + data_index = "fields" + else: + return False + if blacklisted_type in item[data_index].keys(): + content = item[data_index][blacklisted_type] # For every item in the blacklisted array for the type for blacklisted_item in settings.OPENSEARCH_BLACKLISTED[ blacklisted_type @@ -109,7 +115,7 @@ def filter_blacklisted(user, response): # Let the UI know something was redacted if ( "exemption" - not in response["hits"]["hits"][index]["_source"] + not in response["hits"]["hits"][index][data_index] ): response["redacted"] += 1 # Anonymous @@ -120,7 +126,7 @@ def filter_blacklisted(user, response): if not user.is_superuser: response["hits"]["hits"][index] = None else: - response["hits"]["hits"][index]["_source"][ + response["hits"]["hits"][index][data_index][ "exemption" ] = True @@ -128,6 +134,49 @@ def filter_blacklisted(user, response): response["hits"]["hits"] = [hit for hit in response["hits"]["hits"] if hit] +def construct_query(query, size): + """ + Accept some query parameters and construct an OpenSearch query. + """ + if not size: + size = 5 + query = { + "size": size, + "query": { + "bool": { + "must": [ + { + "query_string": { + "query": query, + # "fields": fields, + # "default_field": "msg", + # "type": "best_fields", + "fuzziness": "AUTO", + "fuzzy_transpositions": True, + "fuzzy_max_expansions": 50, + "fuzzy_prefix_length": 0, + # "minimum_should_match": 1, + "default_operator": "or", + "analyzer": "standard", + "lenient": True, + "boost": 1, + "allow_leading_wildcard": True, + # "enable_position_increments": False, + "phrase_slop": 3, + # "max_determinized_states": 10000, + "quote_field_suffix": "", + "quote_analyzer": "standard", + "analyze_wildcard": False, + "auto_generate_synonyms_phrase_query": True, + } + } + ] + } + }, + } + return query + + def run_main_query(client, user, query, custom_query=False, index=None, size=None): """ Low level helper to run an ES query. @@ -153,12 +202,22 @@ def run_main_query(client, user, query, custom_query=False, index=None, size=Non filter_blacklisted(user, response) return response + def parse_results(results): results_parsed = [] if "hits" in results.keys(): if "hits" in results["hits"]: for item in results["hits"]["hits"]: - element = item["_source"] + if "_source" in item.keys(): + data_index = "_source" + elif "fields" in item.keys(): + data_index = "fields" + else: + return False + element = item[data_index] + # Why are fields in lists... + if data_index == "fields": + element = {k: v[0] for k, v in element.items() if len(v)} element["id"] = item["_id"] # Split the timestamp into date and time @@ -173,11 +232,19 @@ def parse_results(results): date = ts_spl[0] time = ts_spl[1] element["date"] = date - element["time"] = time + if "." in time: + time_spl = time.split(".") + if len(time_spl) == 2: + element["time"] = time.split(".")[0] + else: + element["time"] = time + else: + element["time"] = time results_parsed.append(element) return results_parsed -def query_results(request, query_params, size=None): + +def query_results(request, query_params, size=None, annotate=True, custom_query=False): """ API helper to alter the OpenSearch return format into something a bit better to parse. @@ -185,6 +252,7 @@ def query_results(request, query_params, size=None): results with the other data we have. """ # is_anonymous = isinstance(request.user, AnonymousUser) + query = None message = None message_class = None add_bool = [] @@ -202,6 +270,8 @@ def query_results(request, query_params, size=None): message = "Size is not permitted" message_class = "danger" return {"message": message, "class": message_class} + else: + size = 20 if "source" in query_params: source = query_params["source"] if source not in settings.OPENSEARCH_MAIN_SOURCES: @@ -275,72 +345,79 @@ def query_results(request, query_params, size=None): if "query" in query_params: query = query_params["query"] search_query = construct_query(query, size) - if add_bool: - for item in add_bool: - search_query["query"]["bool"]["must"].append({"match": item}) - if add_top: - for item in add_top: - search_query["query"]["bool"]["must"].append(item) - if add_top_negative: - for item in add_top_negative: - if "must_not" in search_query["query"]["bool"]: - search_query["query"]["bool"]["must_not"].append(item) - else: - search_query["query"]["bool"]["must_not"] = [item] - if sort: - search_query["sort"] = sort - - if "index" in query_params: - if not request.user.is_superuser: - message = "How did you get here?" - message_class = "danger" - return {"message": message, "class": message_class} + else: + print("NO QUERY") + if custom_query: + print("CUSTOM") + search_query = custom_query + if add_bool: + for item in add_bool: + search_query["query"]["bool"]["must"].append({"match": item}) + if add_top: + for item in add_top: + search_query["query"]["bool"]["must"].append(item) + if add_top_negative: + for item in add_top_negative: + if "must_not" in search_query["query"]["bool"]: + search_query["query"]["bool"]["must_not"].append(item) else: - index = query_params["index"] - if index == "main": - index = settings.OPENSEARCH_INDEX_MAIN - elif index == "meta": + search_query["query"]["bool"]["must_not"] = [item] + if sort: + search_query["sort"] = sort + + if "index" in query_params: + index = query_params["index"] + if index == "main": + index = settings.OPENSEARCH_INDEX_MAIN + else: + if request.user.is_superuser: + if index == "meta": index = settings.OPENSEARCH_INDEX_META - elif index == "int": + if index == "int": index = settings.OPENSEARCH_INDEX_INT else: message = "Index is not valid." message_class = "danger" return {"message": message, "class": message_class} - else: - index = settings.OPENSEARCH_INDEX_MAIN - results = run_main_query( - client, - request.user, # passed through run_main_query to filter_blacklisted - search_query, - custom_query=True, - index=index, - size=size, - ) - if not results: - return False - if isinstance(results, Exception): - message = results.info["error"]["root_cause"][0]["reason"] - message_class = "danger" - return {"message": message, "class": message_class} - if len(results["hits"]["hits"]) == 0: - message = "No results." - message_class = "danger" - return {"message": message, "class": message_class} - - results_parsed = parse_results(results) + else: + message = "Not permitted to search by this index" + message_class = "danger" + return {"message": message, "class": message_class} + else: + index = settings.OPENSEARCH_INDEX_MAIN + results = run_main_query( + client, + request.user, # passed through run_main_query to filter_blacklisted + search_query, + custom_query=True, + index=index, + size=size, + ) + if not results: + return False + if isinstance(results, Exception): + message = results.info["error"]["root_cause"][0]["reason"] + message_class = "danger" + return {"message": message, "class": message_class} + if len(results["hits"]["hits"]) == 0: + message = "No results." + message_class = "danger" + return {"message": message, "class": message_class} + results_parsed = parse_results(results) + if annotate: annotate_results(results_parsed) - context = { - "query": query, - "object_list": results_parsed, - "card": results["hits"]["total"]["value"], - "took": results["took"], - "redacted": results["redacted"], - "exemption": results["exemption"], - } - return context + context = { + "object_list": results_parsed, + "card": results["hits"]["total"]["value"], + "took": results["took"], + "redacted": results["redacted"], + "exemption": results["exemption"], + } + if query: + context["query"] = query + return context def query_single_result(request): @@ -355,46 +432,3 @@ def query_single_result(request): context["item"] = context["object_list"][0] return context - - -def construct_query(query, size): - """ - Accept some query parameters and construct an OpenSearch query. - """ - if not size: - size = 5 - query = { - "size": size, - "query": { - "bool": { - "must": [ - { - "query_string": { - "query": query, - # "fields": fields, - # "default_field": "msg", - # "type": "best_fields", - "fuzziness": "AUTO", - "fuzzy_transpositions": True, - "fuzzy_max_expansions": 50, - "fuzzy_prefix_length": 0, - # "minimum_should_match": 1, - "default_operator": "or", - "analyzer": "standard", - "lenient": True, - "boost": 1, - "allow_leading_wildcard": True, - # "enable_position_increments": False, - "phrase_slop": 3, - # "max_determinized_states": 10000, - "quote_field_suffix": "", - "quote_analyzer": "standard", - "analyze_wildcard": False, - "auto_generate_synonyms_phrase_query": True, - } - } - ] - } - }, - } - return query diff --git a/core/static/js/column-shifter.js b/core/static/js/column-shifter.js index 6ccb5f7..3fc9f14 100644 --- a/core/static/js/column-shifter.js +++ b/core/static/js/column-shifter.js @@ -122,7 +122,7 @@ $(document).ready(function(){ // Show table content and hide spiner var show_table_content = function($table_class_container){ $table_class_container.find("#loader").hide(); - $table_class_container.find(".table-container").show(); + $table_class_container.find("#table-container").show(); const event = new Event('restore-scroll'); document.dispatchEvent(event); }; @@ -222,7 +222,7 @@ $(document).ready(function(){ if(idx==undefined) { idx = 0; } - return $('.table-container').eq(idx).find('.btn-shift-column').filter(function(z) { + return $('#table-container').eq(idx).find('.btn-shift-column').filter(function(z) { return $(this).data('state')=='off' }).map(function(z) { return $(this).data('td-class') diff --git a/core/static/modal.js b/core/static/modal.js index dd48b7a..bf7db64 100644 --- a/core/static/modal.js +++ b/core/static/modal.js @@ -2,12 +2,20 @@ var modal = document.getElementById("modal"); var html = document.querySelector('html'); +var disableModal = function() { + modal.classList.remove('is-active'); + html.classList.remove('is-clipped'); + var modal_refresh = document.getElementsByClassName("modal-refresh"); + for(var i = 0; i < modal_refresh.length; i++) { + modal_refresh[i].remove(); +} +} + var elements = document.querySelectorAll('.modal-background'); for(var i = 0; i < elements.length; i++) { elements[i].addEventListener('click', function(e) { // elements[i].preventDefault(); - modal.classList.remove('is-active'); - html.classList.remove('is-clipped'); + disableModal(); }); } @@ -15,8 +23,7 @@ var elements = document.querySelectorAll('.modal-close'); for(var i = 0; i < elements.length; i++) { elements[i].addEventListener('click', function(e) { // elements[i].preventDefault(); - modal.classList.remove('is-active'); - html.classList.remove('is-clipped'); + disableModal(); }); } @@ -25,8 +32,7 @@ function activateButtons() { for(var i = 0; i < elements.length; i++) { elements[i].addEventListener('click', function(e) { // elements[i].preventDefault(); - modal.classList.remove('is-active'); - html.classList.remove('is-clipped'); + disableModal(); }); } } diff --git a/core/templates/modals/context.html b/core/templates/modals/context.html index 05159f9..8f61505 100644 --- a/core/templates/modals/context.html +++ b/core/templates/modals/context.html @@ -4,10 +4,44 @@ + + + + +