From 3f02c614637f7be88ee0cbeda02283a56234bbdf Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Fri, 26 Aug 2022 17:16:55 +0100 Subject: [PATCH] Improve data security by mandating token search --- app/local_settings.example.py | 39 +++++ core/lib/opensearch.py | 149 +++++++++++++----- core/static/css/bulma-switch.min.css | 1 + core/templates/base.html | 1 + core/templates/ui/drilldown/drilldown.html | 112 ++++++------- .../templates/ui/drilldown/table_results.html | 20 --- .../ui/drilldown/table_results_partial.html | 2 +- core/views/helpers.py | 60 +++++-- core/views/ui/drilldown.py | 68 ++++++-- 9 files changed, 309 insertions(+), 143 deletions(-) create mode 100644 core/static/css/bulma-switch.min.css diff --git a/app/local_settings.example.py b/app/local_settings.example.py index 141bac4..b0dbf65 100644 --- a/app/local_settings.example.py +++ b/app/local_settings.example.py @@ -16,6 +16,45 @@ OPENSEARCH_MAIN_WILDCARD_ANON = False OPENSEARCH_MAIN_SOURCES = ["irc", "dis", "all"] DRILLDOWN_RESULTS_PER_PAGE = 15 +# Encryption +ENCRYPTION = False +ENCRYPTION_KEY = b"" + +# Hashing +HASHING = True +HASHING_KEY = "xxx" + +# Common to encryption and hashing +WHITELIST_FIELDS = [ + "ts", + "date", + "time", + "sentiment", + "version_sentiment", + "tokens", + "num_chans", + "num_users", + "tokens", + "src", + "exemption", + "hidden", +] + +# Don't obfuscate these parameters, or lookup hashes in them +NO_OBFUSCATE_PARAMS = [ + "query", + "query_full", + "size", + "source", + "sorting", + "tags", + "index", + "dedup", + "check_sentiment", + "sentiment_method", + "dates", +] + OPENSEARCH_BLACKLISTED = {} # URLs diff --git a/core/lib/opensearch.py b/core/lib/opensearch.py index bd8570b..a60f0a3 100644 --- a/core/lib/opensearch.py +++ b/core/lib/opensearch.py @@ -5,7 +5,16 @@ from opensearchpy import OpenSearch from opensearchpy.exceptions import NotFoundError, RequestError from core.lib.threshold import annotate_num_chans, annotate_num_users, annotate_online -from core.views.helpers import dedup_list, encrypt_list, hash_list, hash_lookup +from core.views.helpers import ( + SearchDenied, + dedup_list, + encrypt_list, + hash_list, + hash_lookup, +) + +# from json import dumps +# pp = lambda x: print(dumps(x, indent=2)) def initialise_opensearch(): @@ -141,47 +150,66 @@ def filter_blacklisted(user, response): response["hits"]["hits"] = [hit for hit in response["hits"]["hits"] if hit] -def construct_query(query, size): +def construct_query(query, size, use_query_string=True, tokens=False): """ Accept some query parameters and construct an OpenSearch query. """ if not size: size = 5 - query = { + query_base = { "size": size, - "query": { - "bool": { - "must": [ - { - "query_string": { - "query": query, - # "fields": fields, - # "default_field": "msg", - # "type": "best_fields", - "fuzziness": "AUTO", - "fuzzy_transpositions": True, - "fuzzy_max_expansions": 50, - "fuzzy_prefix_length": 0, - # "minimum_should_match": 1, - "default_operator": "or", - "analyzer": "standard", - "lenient": True, - "boost": 1, - "allow_leading_wildcard": True, - # "enable_position_increments": False, - "phrase_slop": 3, - # "max_determinized_states": 10000, - "quote_field_suffix": "", - "quote_analyzer": "standard", - "analyze_wildcard": False, - "auto_generate_synonyms_phrase_query": True, - } - } - ] - } - }, + "query": {"bool": {"must": []}}, } - return query + query_string = { + "query_string": { + "query": query, + # "fields": fields, + # "default_field": "msg", + # "type": "best_fields", + "fuzziness": "AUTO", + "fuzzy_transpositions": True, + "fuzzy_max_expansions": 50, + "fuzzy_prefix_length": 0, + # "minimum_should_match": 1, + "default_operator": "or", + "analyzer": "standard", + "lenient": True, + "boost": 1, + "allow_leading_wildcard": True, + # "enable_position_increments": False, + "phrase_slop": 3, + # "max_determinized_states": 10000, + "quote_field_suffix": "", + "quote_analyzer": "standard", + "analyze_wildcard": False, + "auto_generate_synonyms_phrase_query": True, + } + } + query_tokens = { + "simple_query_string": { + # "tokens": query, + "query": query, + "fields": ["tokens"], + "flags": "ALL", + "fuzzy_transpositions": True, + "fuzzy_max_expansions": 50, + "fuzzy_prefix_length": 0, + "default_operator": "and", + "analyzer": "standard", + "lenient": True, + "boost": 1, + "quote_field_suffix": "", + "analyze_wildcard": False, + "auto_generate_synonyms_phrase_query": False, + } + } + if tokens: + query_base["query"]["bool"]["must"].append(query_tokens) + # query["query"]["bool"]["must"].append(query_string) + # query["query"]["bool"]["must"][0]["query_string"]["fields"] = ["tokens"] + elif use_query_string: + query_base["query"]["bool"]["must"].append(query_string) + return query_base def run_main_query(client, user, query, custom_query=False, index=None, size=None): @@ -261,6 +289,7 @@ def query_results( dedup=False, dedup_fields=None, lookup_hashes=True, + tags=None, ): """ API helper to alter the OpenSearch return format into something @@ -276,12 +305,15 @@ def query_results( add_top = [] add_top_negative = [] sort = None + query_created = False # Lookup the hash values but don't disclose them to the user if lookup_hashes: if settings.HASHING: query_params = deepcopy(query_params) - hash_lookup(query_params) + hash_lookup(request.user, query_params) + if tags: + hash_lookup(request.user, tags) if request.user.is_anonymous: sizes = settings.OPENSEARCH_MAIN_SIZES_ANON @@ -366,15 +398,53 @@ def query_results( range_query_precise["match"]["sentiment"] = 0 add_top_negative.append(range_query_precise) + # Only one of query or query_full can be active at once + # We prefer query because it's simpler if "query" in query_params: query = query_params["query"] - search_query = construct_query(query, size) + search_query = construct_query(query, size, tokens=True) + query_created = True + elif "query_full" in query_params: + query_full = query_params["query_full"] + if request.user.has_perm("query_search"): + search_query = construct_query(query_full, size) + query_created = True + else: + message = "You cannot search by query string" + message_class = "danger" + return {"message": message, "class": message_class} else: if custom_query: search_query = custom_query + + if tags: + # Get a blank search query + if not query_created: + search_query = construct_query(None, size, use_query_string=False) + query_created = True + for tagname, tagvalue in tags.items(): + add_bool.append({tagname: tagvalue}) + + required_any = ["query_full", "query", "tags"] + if not any([field in query_params.keys() for field in required_any]): + if not custom_query: + message = "Empty query!" + message_class = "warning" + return {"message": message, "class": message_class} + if add_bool: + # if "bool" not in search_query["query"]: + # search_query["query"]["bool"] = {} + # if "must" not in search_query["query"]["bool"]: + # search_query["query"]["bool"] = {"must": []} + for item in add_bool: - search_query["query"]["bool"]["must"].append({"match": item}) + k, v = list(item.items())[0] + if isinstance(v, SearchDenied): + message = f"Access denied: search by protected field {k}: {v.value}" + message_class = "danger" + return {"message": message, "class": message_class} + search_query["query"]["bool"]["must"].append({"match_phrase": item}) if add_top: for item in add_top: search_query["query"]["bool"]["must"].append(item) @@ -398,7 +468,6 @@ def query_results( return { "message": message, "class": message_class, - "params": query_params, } if index == "meta": index = settings.OPENSEARCH_INDEX_META @@ -410,7 +479,6 @@ def query_results( return { "message": message, "class": message_class, - "params": query_params, } else: @@ -461,7 +529,6 @@ def query_results( if not request.user.has_perm("view_plain"): if settings.HASHING: hash_list(request.user, results_parsed) - # process_list(reqults) # IMPORTANT! - DO NOT PASS query_params to the user! diff --git a/core/static/css/bulma-switch.min.css b/core/static/css/bulma-switch.min.css new file mode 100644 index 0000000..5a07523 --- /dev/null +++ b/core/static/css/bulma-switch.min.css @@ -0,0 +1 @@ +.switch[type=checkbox]{outline:0;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none;display:inline-block;position:absolute;opacity:0}.switch[type=checkbox]:focus+label::after,.switch[type=checkbox]:focus+label::before,.switch[type=checkbox]:focus+label:after,.switch[type=checkbox]:focus+label:before{outline:1px dotted #b5b5b5}.switch[type=checkbox][disabled]{cursor:not-allowed}.switch[type=checkbox][disabled]+label{opacity:.5}.switch[type=checkbox][disabled]+label::before,.switch[type=checkbox][disabled]+label:before{opacity:.5}.switch[type=checkbox][disabled]+label::after,.switch[type=checkbox][disabled]+label:after{opacity:.5}.switch[type=checkbox][disabled]+label:hover{cursor:not-allowed}.switch[type=checkbox]+label{position:relative;display:inline-flex;align-items:center;justify-content:flex-start;font-size:1rem;height:2.5em;line-height:1.5;padding-left:3.5rem;padding-top:.2rem;cursor:pointer}.switch[type=checkbox]+label::before,.switch[type=checkbox]+label:before{position:absolute;display:block;top:calc(50% - 1.5rem * .5);left:0;width:3rem;height:1.5rem;border:.1rem solid transparent;border-radius:4px;background:#b5b5b5;content:""}.switch[type=checkbox]+label::after,.switch[type=checkbox]+label:after{display:block;position:absolute;top:calc(50% - 1rem * .5);left:.25rem;width:1rem;height:1rem;transform:translate3d(0,0,0);border-radius:4px;background:#fff;transition:all .25s ease-out;content:""}.switch[type=checkbox]+label .switch-active,.switch[type=checkbox]+label .switch-inactive{font-size:.9rem;z-index:1;margin-top:-4px}.switch[type=checkbox]+label.has-text-inside .switch-inactive{margin-left:-1.925rem}.switch[type=checkbox]+label.has-text-inside .switch-active{margin-left:-3.25rem}.switch[type=checkbox].is-rtl+label{padding-left:0;padding-right:3.5rem}.switch[type=checkbox].is-rtl+label::before,.switch[type=checkbox].is-rtl+label:before{left:auto;right:0}.switch[type=checkbox].is-rtl+label::after,.switch[type=checkbox].is-rtl+label:after{left:auto;right:1.625rem}.switch[type=checkbox]:checked+label::before,.switch[type=checkbox]:checked+label:before{background:#00d1b2}.switch[type=checkbox]:checked+label::after{left:1.625rem}.switch[type=checkbox]:checked.is-rtl+label::after,.switch[type=checkbox]:checked.is-rtl+label:after{left:auto;right:.25rem}.switch[type=checkbox].is-outlined+label::before,.switch[type=checkbox].is-outlined+label:before{background-color:transparent;border-color:#b5b5b5}.switch[type=checkbox].is-outlined+label::after,.switch[type=checkbox].is-outlined+label:after{background:#b5b5b5}.switch[type=checkbox].is-outlined:checked+label::before,.switch[type=checkbox].is-outlined:checked+label:before{background-color:transparent;border-color:#00d1b2}.switch[type=checkbox].is-outlined:checked+label::after,.switch[type=checkbox].is-outlined:checked+label:after{background:#00d1b2}.switch[type=checkbox].is-thin+label::before,.switch[type=checkbox].is-thin+label:before{top:.5454545456rem;height:.375rem}.switch[type=checkbox].is-thin+label::after,.switch[type=checkbox].is-thin+label:after{box-shadow:0 0 3px #7a7a7a}.switch[type=checkbox].is-rounded+label::before,.switch[type=checkbox].is-rounded+label:before{border-radius:24px}.switch[type=checkbox].is-rounded+label::after,.switch[type=checkbox].is-rounded+label:after{border-radius:50%}.switch[type=checkbox].is-small+label{position:relative;display:inline-flex;align-items:center;justify-content:flex-start;font-size:.75rem;height:2.5em;line-height:1.5;padding-left:2.75rem;padding-top:.2rem;cursor:pointer}.switch[type=checkbox].is-small+label::before,.switch[type=checkbox].is-small+label:before{position:absolute;display:block;top:calc(50% - 1.125rem * .5);left:0;width:2.25rem;height:1.125rem;border:.1rem solid transparent;border-radius:4px;background:#b5b5b5;content:""}.switch[type=checkbox].is-small+label::after,.switch[type=checkbox].is-small+label:after{display:block;position:absolute;top:calc(50% - .625rem * .5);left:.25rem;width:.625rem;height:.625rem;transform:translate3d(0,0,0);border-radius:4px;background:#fff;transition:all .25s ease-out;content:""}.switch[type=checkbox].is-small+label .switch-active,.switch[type=checkbox].is-small+label .switch-inactive{font-size:.65rem;z-index:1;margin-top:-4px}.switch[type=checkbox].is-small+label.has-text-inside .switch-inactive{margin-left:-1.55rem}.switch[type=checkbox].is-small+label.has-text-inside .switch-active{margin-left:-2.5rem}.switch[type=checkbox].is-small.is-rtl+label{padding-left:0;padding-right:2.75rem}.switch[type=checkbox].is-small.is-rtl+label::before,.switch[type=checkbox].is-small.is-rtl+label:before{left:auto;right:0}.switch[type=checkbox].is-small.is-rtl+label::after,.switch[type=checkbox].is-small.is-rtl+label:after{left:auto;right:1.25rem}.switch[type=checkbox].is-small:checked+label::before,.switch[type=checkbox].is-small:checked+label:before{background:#00d1b2}.switch[type=checkbox].is-small:checked+label::after{left:1.25rem}.switch[type=checkbox].is-small:checked.is-rtl+label::after,.switch[type=checkbox].is-small:checked.is-rtl+label:after{left:auto;right:.25rem}.switch[type=checkbox].is-small.is-outlined+label::before,.switch[type=checkbox].is-small.is-outlined+label:before{background-color:transparent;border-color:#b5b5b5}.switch[type=checkbox].is-small.is-outlined+label::after,.switch[type=checkbox].is-small.is-outlined+label:after{background:#b5b5b5}.switch[type=checkbox].is-small.is-outlined:checked+label::before,.switch[type=checkbox].is-small.is-outlined:checked+label:before{background-color:transparent;border-color:#00d1b2}.switch[type=checkbox].is-small.is-outlined:checked+label::after,.switch[type=checkbox].is-small.is-outlined:checked+label:after{background:#00d1b2}.switch[type=checkbox].is-small.is-thin+label::before,.switch[type=checkbox].is-small.is-thin+label:before{top:.4090909093rem;height:.28125rem}.switch[type=checkbox].is-small.is-thin+label::after,.switch[type=checkbox].is-small.is-thin+label:after{box-shadow:0 0 3px #7a7a7a}.switch[type=checkbox].is-small.is-rounded+label::before,.switch[type=checkbox].is-small.is-rounded+label:before{border-radius:24px}.switch[type=checkbox].is-small.is-rounded+label::after,.switch[type=checkbox].is-small.is-rounded+label:after{border-radius:50%}.switch[type=checkbox].is-medium+label{position:relative;display:inline-flex;align-items:center;justify-content:flex-start;font-size:1.25rem;height:2.5em;line-height:1.5;padding-left:4.25rem;padding-top:.2rem;cursor:pointer}.switch[type=checkbox].is-medium+label::before,.switch[type=checkbox].is-medium+label:before{position:absolute;display:block;top:calc(50% - 1.875rem * .5);left:0;width:3.75rem;height:1.875rem;border:.1rem solid transparent;border-radius:4px;background:#b5b5b5;content:""}.switch[type=checkbox].is-medium+label::after,.switch[type=checkbox].is-medium+label:after{display:block;position:absolute;top:calc(50% - 1.375rem * .5);left:.25rem;width:1.375rem;height:1.375rem;transform:translate3d(0,0,0);border-radius:4px;background:#fff;transition:all .25s ease-out;content:""}.switch[type=checkbox].is-medium+label .switch-active,.switch[type=checkbox].is-medium+label .switch-inactive{font-size:1.15rem;z-index:1;margin-top:-4px}.switch[type=checkbox].is-medium+label.has-text-inside .switch-inactive{margin-left:-2.3rem}.switch[type=checkbox].is-medium+label.has-text-inside .switch-active{margin-left:-4rem}.switch[type=checkbox].is-medium.is-rtl+label{padding-left:0;padding-right:4.25rem}.switch[type=checkbox].is-medium.is-rtl+label::before,.switch[type=checkbox].is-medium.is-rtl+label:before{left:auto;right:0}.switch[type=checkbox].is-medium.is-rtl+label::after,.switch[type=checkbox].is-medium.is-rtl+label:after{left:auto;right:2rem}.switch[type=checkbox].is-medium:checked+label::before,.switch[type=checkbox].is-medium:checked+label:before{background:#00d1b2}.switch[type=checkbox].is-medium:checked+label::after{left:2rem}.switch[type=checkbox].is-medium:checked.is-rtl+label::after,.switch[type=checkbox].is-medium:checked.is-rtl+label:after{left:auto;right:.25rem}.switch[type=checkbox].is-medium.is-outlined+label::before,.switch[type=checkbox].is-medium.is-outlined+label:before{background-color:transparent;border-color:#b5b5b5}.switch[type=checkbox].is-medium.is-outlined+label::after,.switch[type=checkbox].is-medium.is-outlined+label:after{background:#b5b5b5}.switch[type=checkbox].is-medium.is-outlined:checked+label::before,.switch[type=checkbox].is-medium.is-outlined:checked+label:before{background-color:transparent;border-color:#00d1b2}.switch[type=checkbox].is-medium.is-outlined:checked+label::after,.switch[type=checkbox].is-medium.is-outlined:checked+label:after{background:#00d1b2}.switch[type=checkbox].is-medium.is-thin+label::before,.switch[type=checkbox].is-medium.is-thin+label:before{top:.6818181819rem;height:.46875rem}.switch[type=checkbox].is-medium.is-thin+label::after,.switch[type=checkbox].is-medium.is-thin+label:after{box-shadow:0 0 3px #7a7a7a}.switch[type=checkbox].is-medium.is-rounded+label::before,.switch[type=checkbox].is-medium.is-rounded+label:before{border-radius:24px}.switch[type=checkbox].is-medium.is-rounded+label::after,.switch[type=checkbox].is-medium.is-rounded+label:after{border-radius:50%}.switch[type=checkbox].is-large+label{position:relative;display:inline-flex;align-items:center;justify-content:flex-start;font-size:1.5rem;height:2.5em;line-height:1.5;padding-left:5rem;padding-top:.2rem;cursor:pointer}.switch[type=checkbox].is-large+label::before,.switch[type=checkbox].is-large+label:before{position:absolute;display:block;top:calc(50% - 2.25rem * .5);left:0;width:4.5rem;height:2.25rem;border:.1rem solid transparent;border-radius:4px;background:#b5b5b5;content:""}.switch[type=checkbox].is-large+label::after,.switch[type=checkbox].is-large+label:after{display:block;position:absolute;top:calc(50% - 1.75rem * .5);left:.25rem;width:1.75rem;height:1.75rem;transform:translate3d(0,0,0);border-radius:4px;background:#fff;transition:all .25s ease-out;content:""}.switch[type=checkbox].is-large+label .switch-active,.switch[type=checkbox].is-large+label .switch-inactive{font-size:1.4rem;z-index:1;margin-top:-4px}.switch[type=checkbox].is-large+label.has-text-inside .switch-inactive{margin-left:-2.675rem}.switch[type=checkbox].is-large+label.has-text-inside .switch-active{margin-left:-4.75rem}.switch[type=checkbox].is-large.is-rtl+label{padding-left:0;padding-right:5rem}.switch[type=checkbox].is-large.is-rtl+label::before,.switch[type=checkbox].is-large.is-rtl+label:before{left:auto;right:0}.switch[type=checkbox].is-large.is-rtl+label::after,.switch[type=checkbox].is-large.is-rtl+label:after{left:auto;right:2.375rem}.switch[type=checkbox].is-large:checked+label::before,.switch[type=checkbox].is-large:checked+label:before{background:#00d1b2}.switch[type=checkbox].is-large:checked+label::after{left:2.375rem}.switch[type=checkbox].is-large:checked.is-rtl+label::after,.switch[type=checkbox].is-large:checked.is-rtl+label:after{left:auto;right:.25rem}.switch[type=checkbox].is-large.is-outlined+label::before,.switch[type=checkbox].is-large.is-outlined+label:before{background-color:transparent;border-color:#b5b5b5}.switch[type=checkbox].is-large.is-outlined+label::after,.switch[type=checkbox].is-large.is-outlined+label:after{background:#b5b5b5}.switch[type=checkbox].is-large.is-outlined:checked+label::before,.switch[type=checkbox].is-large.is-outlined:checked+label:before{background-color:transparent;border-color:#00d1b2}.switch[type=checkbox].is-large.is-outlined:checked+label::after,.switch[type=checkbox].is-large.is-outlined:checked+label:after{background:#00d1b2}.switch[type=checkbox].is-large.is-thin+label::before,.switch[type=checkbox].is-large.is-thin+label:before{top:.8181818183rem;height:.5625rem}.switch[type=checkbox].is-large.is-thin+label::after,.switch[type=checkbox].is-large.is-thin+label:after{box-shadow:0 0 3px #7a7a7a}.switch[type=checkbox].is-large.is-rounded+label::before,.switch[type=checkbox].is-large.is-rounded+label:before{border-radius:24px}.switch[type=checkbox].is-large.is-rounded+label::after,.switch[type=checkbox].is-large.is-rounded+label:after{border-radius:50%}.switch[type=checkbox].is-white+label .switch-active{display:none}.switch[type=checkbox].is-white+label .switch-inactive{display:inline-block}.switch[type=checkbox].is-white:checked+label::before,.switch[type=checkbox].is-white:checked+label:before{background:#fff}.switch[type=checkbox].is-white:checked+label .switch-active{display:inline-block}.switch[type=checkbox].is-white:checked+label .switch-inactive{display:none}.switch[type=checkbox].is-white.is-outlined:checked+label::before,.switch[type=checkbox].is-white.is-outlined:checked+label:before{background-color:transparent;border-color:#fff!important}.switch[type=checkbox].is-white.is-outlined:checked+label::after,.switch[type=checkbox].is-white.is-outlined:checked+label:after{background:#fff}.switch[type=checkbox].is-white.is-thin.is-outlined+label::after,.switch[type=checkbox].is-white.is-thin.is-outlined+label:after{box-shadow:none}.switch[type=checkbox].is-unchecked-white+label::before,.switch[type=checkbox].is-unchecked-white+label:before{background:#fff}.switch[type=checkbox].is-unchecked-white.is-outlined+label::before,.switch[type=checkbox].is-unchecked-white.is-outlined+label:before{background-color:transparent;border-color:#fff!important}.switch[type=checkbox].is-unchecked-white.is-outlined+label::after,.switch[type=checkbox].is-unchecked-white.is-outlined+label:after{background:#fff}.switch[type=checkbox].is-black+label .switch-active{display:none}.switch[type=checkbox].is-black+label .switch-inactive{display:inline-block}.switch[type=checkbox].is-black:checked+label::before,.switch[type=checkbox].is-black:checked+label:before{background:#0a0a0a}.switch[type=checkbox].is-black:checked+label .switch-active{display:inline-block}.switch[type=checkbox].is-black:checked+label .switch-inactive{display:none}.switch[type=checkbox].is-black.is-outlined:checked+label::before,.switch[type=checkbox].is-black.is-outlined:checked+label:before{background-color:transparent;border-color:#0a0a0a!important}.switch[type=checkbox].is-black.is-outlined:checked+label::after,.switch[type=checkbox].is-black.is-outlined:checked+label:after{background:#0a0a0a}.switch[type=checkbox].is-black.is-thin.is-outlined+label::after,.switch[type=checkbox].is-black.is-thin.is-outlined+label:after{box-shadow:none}.switch[type=checkbox].is-unchecked-black+label::before,.switch[type=checkbox].is-unchecked-black+label:before{background:#0a0a0a}.switch[type=checkbox].is-unchecked-black.is-outlined+label::before,.switch[type=checkbox].is-unchecked-black.is-outlined+label:before{background-color:transparent;border-color:#0a0a0a!important}.switch[type=checkbox].is-unchecked-black.is-outlined+label::after,.switch[type=checkbox].is-unchecked-black.is-outlined+label:after{background:#0a0a0a}.switch[type=checkbox].is-light+label .switch-active{display:none}.switch[type=checkbox].is-light+label .switch-inactive{display:inline-block}.switch[type=checkbox].is-light:checked+label::before,.switch[type=checkbox].is-light:checked+label:before{background:#f5f5f5}.switch[type=checkbox].is-light:checked+label .switch-active{display:inline-block}.switch[type=checkbox].is-light:checked+label .switch-inactive{display:none}.switch[type=checkbox].is-light.is-outlined:checked+label::before,.switch[type=checkbox].is-light.is-outlined:checked+label:before{background-color:transparent;border-color:#f5f5f5!important}.switch[type=checkbox].is-light.is-outlined:checked+label::after,.switch[type=checkbox].is-light.is-outlined:checked+label:after{background:#f5f5f5}.switch[type=checkbox].is-light.is-thin.is-outlined+label::after,.switch[type=checkbox].is-light.is-thin.is-outlined+label:after{box-shadow:none}.switch[type=checkbox].is-unchecked-light+label::before,.switch[type=checkbox].is-unchecked-light+label:before{background:#f5f5f5}.switch[type=checkbox].is-unchecked-light.is-outlined+label::before,.switch[type=checkbox].is-unchecked-light.is-outlined+label:before{background-color:transparent;border-color:#f5f5f5!important}.switch[type=checkbox].is-unchecked-light.is-outlined+label::after,.switch[type=checkbox].is-unchecked-light.is-outlined+label:after{background:#f5f5f5}.switch[type=checkbox].is-dark+label .switch-active{display:none}.switch[type=checkbox].is-dark+label .switch-inactive{display:inline-block}.switch[type=checkbox].is-dark:checked+label::before,.switch[type=checkbox].is-dark:checked+label:before{background:#363636}.switch[type=checkbox].is-dark:checked+label .switch-active{display:inline-block}.switch[type=checkbox].is-dark:checked+label .switch-inactive{display:none}.switch[type=checkbox].is-dark.is-outlined:checked+label::before,.switch[type=checkbox].is-dark.is-outlined:checked+label:before{background-color:transparent;border-color:#363636!important}.switch[type=checkbox].is-dark.is-outlined:checked+label::after,.switch[type=checkbox].is-dark.is-outlined:checked+label:after{background:#363636}.switch[type=checkbox].is-dark.is-thin.is-outlined+label::after,.switch[type=checkbox].is-dark.is-thin.is-outlined+label:after{box-shadow:none}.switch[type=checkbox].is-unchecked-dark+label::before,.switch[type=checkbox].is-unchecked-dark+label:before{background:#363636}.switch[type=checkbox].is-unchecked-dark.is-outlined+label::before,.switch[type=checkbox].is-unchecked-dark.is-outlined+label:before{background-color:transparent;border-color:#363636!important}.switch[type=checkbox].is-unchecked-dark.is-outlined+label::after,.switch[type=checkbox].is-unchecked-dark.is-outlined+label:after{background:#363636}.switch[type=checkbox].is-primary+label .switch-active{display:none}.switch[type=checkbox].is-primary+label .switch-inactive{display:inline-block}.switch[type=checkbox].is-primary:checked+label::before,.switch[type=checkbox].is-primary:checked+label:before{background:#00d1b2}.switch[type=checkbox].is-primary:checked+label .switch-active{display:inline-block}.switch[type=checkbox].is-primary:checked+label .switch-inactive{display:none}.switch[type=checkbox].is-primary.is-outlined:checked+label::before,.switch[type=checkbox].is-primary.is-outlined:checked+label:before{background-color:transparent;border-color:#00d1b2!important}.switch[type=checkbox].is-primary.is-outlined:checked+label::after,.switch[type=checkbox].is-primary.is-outlined:checked+label:after{background:#00d1b2}.switch[type=checkbox].is-primary.is-thin.is-outlined+label::after,.switch[type=checkbox].is-primary.is-thin.is-outlined+label:after{box-shadow:none}.switch[type=checkbox].is-unchecked-primary+label::before,.switch[type=checkbox].is-unchecked-primary+label:before{background:#00d1b2}.switch[type=checkbox].is-unchecked-primary.is-outlined+label::before,.switch[type=checkbox].is-unchecked-primary.is-outlined+label:before{background-color:transparent;border-color:#00d1b2!important}.switch[type=checkbox].is-unchecked-primary.is-outlined+label::after,.switch[type=checkbox].is-unchecked-primary.is-outlined+label:after{background:#00d1b2}.switch[type=checkbox].is-link+label .switch-active{display:none}.switch[type=checkbox].is-link+label .switch-inactive{display:inline-block}.switch[type=checkbox].is-link:checked+label::before,.switch[type=checkbox].is-link:checked+label:before{background:#485fc7}.switch[type=checkbox].is-link:checked+label .switch-active{display:inline-block}.switch[type=checkbox].is-link:checked+label .switch-inactive{display:none}.switch[type=checkbox].is-link.is-outlined:checked+label::before,.switch[type=checkbox].is-link.is-outlined:checked+label:before{background-color:transparent;border-color:#485fc7!important}.switch[type=checkbox].is-link.is-outlined:checked+label::after,.switch[type=checkbox].is-link.is-outlined:checked+label:after{background:#485fc7}.switch[type=checkbox].is-link.is-thin.is-outlined+label::after,.switch[type=checkbox].is-link.is-thin.is-outlined+label:after{box-shadow:none}.switch[type=checkbox].is-unchecked-link+label::before,.switch[type=checkbox].is-unchecked-link+label:before{background:#485fc7}.switch[type=checkbox].is-unchecked-link.is-outlined+label::before,.switch[type=checkbox].is-unchecked-link.is-outlined+label:before{background-color:transparent;border-color:#485fc7!important}.switch[type=checkbox].is-unchecked-link.is-outlined+label::after,.switch[type=checkbox].is-unchecked-link.is-outlined+label:after{background:#485fc7}.switch[type=checkbox].is-info+label .switch-active{display:none}.switch[type=checkbox].is-info+label .switch-inactive{display:inline-block}.switch[type=checkbox].is-info:checked+label::before,.switch[type=checkbox].is-info:checked+label:before{background:#3e8ed0}.switch[type=checkbox].is-info:checked+label .switch-active{display:inline-block}.switch[type=checkbox].is-info:checked+label .switch-inactive{display:none}.switch[type=checkbox].is-info.is-outlined:checked+label::before,.switch[type=checkbox].is-info.is-outlined:checked+label:before{background-color:transparent;border-color:#3e8ed0!important}.switch[type=checkbox].is-info.is-outlined:checked+label::after,.switch[type=checkbox].is-info.is-outlined:checked+label:after{background:#3e8ed0}.switch[type=checkbox].is-info.is-thin.is-outlined+label::after,.switch[type=checkbox].is-info.is-thin.is-outlined+label:after{box-shadow:none}.switch[type=checkbox].is-unchecked-info+label::before,.switch[type=checkbox].is-unchecked-info+label:before{background:#3e8ed0}.switch[type=checkbox].is-unchecked-info.is-outlined+label::before,.switch[type=checkbox].is-unchecked-info.is-outlined+label:before{background-color:transparent;border-color:#3e8ed0!important}.switch[type=checkbox].is-unchecked-info.is-outlined+label::after,.switch[type=checkbox].is-unchecked-info.is-outlined+label:after{background:#3e8ed0}.switch[type=checkbox].is-success+label .switch-active{display:none}.switch[type=checkbox].is-success+label .switch-inactive{display:inline-block}.switch[type=checkbox].is-success:checked+label::before,.switch[type=checkbox].is-success:checked+label:before{background:#48c78e}.switch[type=checkbox].is-success:checked+label .switch-active{display:inline-block}.switch[type=checkbox].is-success:checked+label .switch-inactive{display:none}.switch[type=checkbox].is-success.is-outlined:checked+label::before,.switch[type=checkbox].is-success.is-outlined:checked+label:before{background-color:transparent;border-color:#48c78e!important}.switch[type=checkbox].is-success.is-outlined:checked+label::after,.switch[type=checkbox].is-success.is-outlined:checked+label:after{background:#48c78e}.switch[type=checkbox].is-success.is-thin.is-outlined+label::after,.switch[type=checkbox].is-success.is-thin.is-outlined+label:after{box-shadow:none}.switch[type=checkbox].is-unchecked-success+label::before,.switch[type=checkbox].is-unchecked-success+label:before{background:#48c78e}.switch[type=checkbox].is-unchecked-success.is-outlined+label::before,.switch[type=checkbox].is-unchecked-success.is-outlined+label:before{background-color:transparent;border-color:#48c78e!important}.switch[type=checkbox].is-unchecked-success.is-outlined+label::after,.switch[type=checkbox].is-unchecked-success.is-outlined+label:after{background:#48c78e}.switch[type=checkbox].is-warning+label .switch-active{display:none}.switch[type=checkbox].is-warning+label .switch-inactive{display:inline-block}.switch[type=checkbox].is-warning:checked+label::before,.switch[type=checkbox].is-warning:checked+label:before{background:#ffe08a}.switch[type=checkbox].is-warning:checked+label .switch-active{display:inline-block}.switch[type=checkbox].is-warning:checked+label .switch-inactive{display:none}.switch[type=checkbox].is-warning.is-outlined:checked+label::before,.switch[type=checkbox].is-warning.is-outlined:checked+label:before{background-color:transparent;border-color:#ffe08a!important}.switch[type=checkbox].is-warning.is-outlined:checked+label::after,.switch[type=checkbox].is-warning.is-outlined:checked+label:after{background:#ffe08a}.switch[type=checkbox].is-warning.is-thin.is-outlined+label::after,.switch[type=checkbox].is-warning.is-thin.is-outlined+label:after{box-shadow:none}.switch[type=checkbox].is-unchecked-warning+label::before,.switch[type=checkbox].is-unchecked-warning+label:before{background:#ffe08a}.switch[type=checkbox].is-unchecked-warning.is-outlined+label::before,.switch[type=checkbox].is-unchecked-warning.is-outlined+label:before{background-color:transparent;border-color:#ffe08a!important}.switch[type=checkbox].is-unchecked-warning.is-outlined+label::after,.switch[type=checkbox].is-unchecked-warning.is-outlined+label:after{background:#ffe08a}.switch[type=checkbox].is-danger+label .switch-active{display:none}.switch[type=checkbox].is-danger+label .switch-inactive{display:inline-block}.switch[type=checkbox].is-danger:checked+label::before,.switch[type=checkbox].is-danger:checked+label:before{background:#f14668}.switch[type=checkbox].is-danger:checked+label .switch-active{display:inline-block}.switch[type=checkbox].is-danger:checked+label .switch-inactive{display:none}.switch[type=checkbox].is-danger.is-outlined:checked+label::before,.switch[type=checkbox].is-danger.is-outlined:checked+label:before{background-color:transparent;border-color:#f14668!important}.switch[type=checkbox].is-danger.is-outlined:checked+label::after,.switch[type=checkbox].is-danger.is-outlined:checked+label:after{background:#f14668}.switch[type=checkbox].is-danger.is-thin.is-outlined+label::after,.switch[type=checkbox].is-danger.is-thin.is-outlined+label:after{box-shadow:none}.switch[type=checkbox].is-unchecked-danger+label::before,.switch[type=checkbox].is-unchecked-danger+label:before{background:#f14668}.switch[type=checkbox].is-unchecked-danger.is-outlined+label::before,.switch[type=checkbox].is-unchecked-danger.is-outlined+label:before{background-color:transparent;border-color:#f14668!important}.switch[type=checkbox].is-unchecked-danger.is-outlined+label::after,.switch[type=checkbox].is-unchecked-danger.is-outlined+label:after{background:#f14668}.field-body .switch[type=checkbox]+label{margin-top:.375em} \ No newline at end of file diff --git a/core/templates/base.html b/core/templates/base.html index 1c6841c..1982d22 100644 --- a/core/templates/base.html +++ b/core/templates/base.html @@ -15,6 +15,7 @@ + diff --git a/core/templates/ui/drilldown/drilldown.html b/core/templates/ui/drilldown/drilldown.html index 30eea32..d4ec2b8 100644 --- a/core/templates/ui/drilldown/drilldown.html +++ b/core/templates/ui/drilldown/drilldown.html @@ -39,60 +39,23 @@ } catch { var value = spl[1]; } - populateSearch(field, value); return `${field}: ${value}`; }); inputTags.BulmaTagsInput().on('after.remove', function(item) { var spl = item.split(": "); var field = spl[0]; var value = spl[1].trim(); - populateSearch(field, value); }); } function populateSearch(field, value) { - var queryElement = document.getElementById('query'); - - var present = true; - if (present == true) { - var combinations = [`${field}: "${value}"`, - `${field}: "${value}"`, - `${field}: ${value}`, - `${field}:${value}`, - `${field}:"${value}"`]; - var toAppend = ` AND ${field}: "${value}"`; - } else { - var combinations = [`NOT ${field}: "${value}"`, - `NOT ${field}: "${value}"`, - `NOT ${field}: ${value}`, - `NOT ${field}:${value}`, - `NOT ${field}:"${value}"`]; - } - var contains = combinations.some(elem => queryElement.value.includes(elem)); - if (!contains) { - queryElement.value+=toAppend; - } else { - for (var index in combinations) { - combination = combinations[index]; - queryElement.value = queryElement.value.replaceAll("AND "+combination, ""); - queryElement.value = queryElement.value.replaceAll(combination, ""); - } - } - - if (field == "src") { - document.getElementById("source").selectedIndex = 2; - } - if (queryElement.value.startsWith(" AND ")) { - queryElement.value = queryElement.value.replace(" AND ", ""); - } - if (queryElement.value.startsWith("AND ")) { - queryElement.value = queryElement.value.replace("AND ", ""); - } + var inputTags = document.getElementById('tags'); + inputTags.BulmaTagsInput().add(field+": "+value); htmx.trigger("#search", "click"); }
{% include 'partials/notify.html' %} -
-
+
+ hx-swap="innerHTML" + name="query" + value="{{ params.query }}" + class="input" + type="text" + placeholder="(science | tech | art) + (interest) -hello"> @@ -166,6 +134,19 @@

+
+
+ + +
+
@@ -411,24 +392,45 @@
+
+
+ + + + +
+
+
+ +
-
- -
- - {% if table %} - {% include 'ui/drilldown/table_results.html' %} - {% else %} - - {% endif %} + {% include 'ui/drilldown/table_results.html' %} +
diff --git a/core/templates/ui/drilldown/table_results.html b/core/templates/ui/drilldown/table_results.html index 07c707d..2ecf004 100644 --- a/core/templates/ui/drilldown/table_results.html +++ b/core/templates/ui/drilldown/table_results.html @@ -37,23 +37,3 @@ {% endif %} {% include 'ui/drilldown/table_results_partial.html' %} {% endif %} -{# Update the tags in case the user changed the query #} -{# Check for focus and refocus #} - \ No newline at end of file diff --git a/core/templates/ui/drilldown/table_results_partial.html b/core/templates/ui/drilldown/table_results_partial.html index 72a5844..1bbe9f1 100644 --- a/core/templates/ui/drilldown/table_results_partial.html +++ b/core/templates/ui/drilldown/table_results_partial.html @@ -143,7 +143,7 @@ {% elif column.name == 'tokens' %} - + {{ cell|joinsep:',' }} {% elif column.name == 'src' %} diff --git a/core/views/helpers.py b/core/views/helpers.py index 9724f0a..3b8ba21 100644 --- a/core/views/helpers.py +++ b/core/views/helpers.py @@ -10,6 +10,11 @@ from sortedcontainers import SortedSet from core import r +class SearchDenied: + def __init__(self, value): + self.value = value + + def dedup_list(data, check_keys): """ Remove duplicate dictionaries from list. @@ -90,7 +95,10 @@ def hash_list(user, data, hash_keys=False): for index, item in enumerate(data_copy): if isinstance(item, dict): for key, value in list(item.items()): - if key not in settings.WHITELIST_FIELDS: + if ( + key not in settings.WHITELIST_FIELDS + and key not in settings.NO_OBFUSCATE_PARAMS + ): if isinstance(value, int): value = str(value) if isinstance(value, bool): @@ -122,18 +130,35 @@ def hash_list(user, data, hash_keys=False): r.hmset(cache, hash_table) -def hash_lookup(data_dict): +def hash_lookup(user, data_dict): cache = "cache.hash" hash_list = SortedSet() - for key, value in data_dict.items(): - if not value: - continue - # hashes = re.findall("\|([^\|]*)\|", value) # noqa - hashes = re.findall("[A-Z0-9]{12,13}", value) - if not hashes: - continue - for hash in hashes: - hash_list.add(hash) + for key, value in list(data_dict.items()): + if ( + key not in settings.WHITELIST_FIELDS + and key not in settings.NO_OBFUSCATE_PARAMS + ): + if not value: + continue + # hashes = re.findall("\|([^\|]*)\|", value) # noqa + if isinstance(value, str): + hashes = re.findall("[A-Z0-9]{12,13}", value) + elif isinstance(value, dict): + hashes = [] + for key, value in value.items(): + if not value: + continue + hashes_iter = re.findall("[A-Z0-9]{12,13}", value) + for h in hashes_iter: + hashes.append(h) + if not hashes: + # Otherwise the user could inject plaintext search queries + if not user.has_perm("bypass_hashing"): + data_dict[key] = SearchDenied(value=data_dict[key]) + # del data_dict[key] + + for hash in hashes: + hash_list.add(hash) if hash_list: values = r.hmget(cache, *hash_list) @@ -147,8 +172,17 @@ def hash_lookup(data_dict): for key in data_dict.keys(): for hash in total: if data_dict[key]: - if hash in data_dict[key]: - data_dict[key] = data_dict[key].replace(f"{hash}", total[hash]) + if isinstance(data_dict[key], str): + if hash in data_dict[key]: + print("Replacing", data_dict[key], "with", total[hash]) + data_dict[key] = data_dict[key].replace( + f"{hash}", total[hash] + ) + elif isinstance(data_dict[key], dict): + for k2, v2 in data_dict[key].items(): + if hash in v2: + print("Replacing", v2, "with", total[hash]) + data_dict[key][k2] = v2.replace(f"{hash}", total[hash]) def encrypt_list(user, data, secret): diff --git a/core/views/ui/drilldown.py b/core/views/ui/drilldown.py index 6671346..8dfb273 100644 --- a/core/views/ui/drilldown.py +++ b/core/views/ui/drilldown.py @@ -55,6 +55,21 @@ def create_tags(query): return tags +def parse_tags(tags_pre): + """ + Parse the tags from the variable tags_pre. + """ + tags = {} + tags_spl = tags_pre.split(",") + if tags_spl: + for tag in tags_spl: + tag = tag.split(": ") + if len(tag) == 2: + key, val = tag + tags[key] = val + return tags + + def make_table(context): table = DrilldownTable(context["object_list"]) context["table"] = table @@ -78,6 +93,8 @@ def make_graph(results): def drilldown_search(request, return_context=False, template=None): + extra_params = {} + if not template: template_name = "ui/drilldown/table_results.html" else: @@ -105,6 +122,10 @@ def drilldown_search(request, return_context=False, template=None): query_params.update(tmp_post) query_params.update(tmp_get) + # URI we're passing to the template for linking + if "csrfmiddlewaretoken" in query_params: + del query_params["csrfmiddlewaretoken"] + # Parse the dates if "dates" in query_params: dates = parse_dates(query_params["dates"]) @@ -118,24 +139,34 @@ def drilldown_search(request, return_context=False, template=None): query_params["to_time"] = dates["to_time"] if "query" in query_params: - context = query_results(request, query_params) + # Remove null values + if query_params["query"] == "": + del query_params["query"] + # Turn the query into tags for populating the taglist + # tags = create_tags(query_params["query"]) + # context["tags"] = tags + # else: + # context = {"object_list": []} - # Turn the query into tags for populating the taglist - tags = create_tags(query_params["query"]) - context["tags"] = tags - else: - context = {"object_list": []} + # Remove null values + if "query_full" in query_params: + if query_params["query_full"] == "": + del query_params["query_full"] + + if "tags" in query_params: + if query_params["tags"] == "": + del query_params["tags"] + else: + tags = parse_tags(query_params["tags"]) + extra_params["tags"] = tags + + context = query_results(request, query_params, **extra_params) # Valid sizes context["sizes"] = sizes - # URI we're passing to the template for linking - if "csrfmiddlewaretoken" in query_params: - del query_params["csrfmiddlewaretoken"] - url_params = urllib.parse.urlencode(query_params) context["client_uri"] = url_params - context["params"] = query_params if "message" in context: response = render(request, template_name, context) @@ -158,6 +189,17 @@ def drilldown_search(request, return_context=False, template=None): clean_url_params = urllib.parse.urlencode(clean_params) context["uri"] = clean_url_params + # Warn users trying to use query string that the simple query supersedes it + if all([x in query_params for x in ["query", "query_full"]]): + context["message"] = ( + "You are searching with both query types. " + "The simple query will be used. " + "The full query will be ignored. " + "Remove the text from the simple query if you wish " + "to use the full query." + ) + context["class"] = "warning" + response = render(request, template_name, context) if request.GET: if request.htmx: @@ -260,7 +302,7 @@ class DrilldownContextModal(APIView): # Lookup the hash values but don't disclose them to the user if settings.HASHING: SAFE_PARAMS = deepcopy(query_params) - hash_lookup(SAFE_PARAMS) + hash_lookup(request.user, SAFE_PARAMS) else: SAFE_PARAMS = query_params @@ -383,7 +425,7 @@ class ThresholdInfoModal(APIView): # Lookup the hash values but don't disclose them to the user if settings.HASHING: SAFE_PARAMS = request.data.dict() - hash_lookup(SAFE_PARAMS) + hash_lookup(request.user, SAFE_PARAMS) safe_net = SAFE_PARAMS["net"] safe_nick = SAFE_PARAMS["nick"] safe_channel = SAFE_PARAMS["channel"]