diff --git a/app/local_settings.example.py b/app/local_settings.example.py index 6e2169e..e330b23 100644 --- a/app/local_settings.example.py +++ b/app/local_settings.example.py @@ -38,6 +38,10 @@ OBFUSCATE_COLON_NUM = 1 SEARCH_FIELDS_DENY = ["ts", "date", "time"] +DELAY_RESULTS = True +# Delay results by this many days +DELAY_DURATION = 10 + # Common to encryption and hashing WHITELIST_FIELDS = [ "ts", @@ -58,7 +62,7 @@ WHITELIST_FIELDS = [ # Don't obfuscate these parameters, or lookup hashes in them NO_OBFUSCATE_PARAMS = [ "query", -# "query_full", + # "query_full", "size", "source", "sorting", diff --git a/core/lib/opensearch.py b/core/lib/opensearch.py index ccee8eb..3a3be8e 100644 --- a/core/lib/opensearch.py +++ b/core/lib/opensearch.py @@ -1,4 +1,5 @@ from copy import deepcopy +from datetime import datetime, timedelta from django.conf import settings from opensearchpy import OpenSearch @@ -327,8 +328,8 @@ def query_results( elif isinstance(x, LookupDenied): message += f"Lookup({x.key}: {x.value}) " if denied: - #message = [f"{i}" for i in message] - #message = "\n".join(message) + # message = [f"{i}" for i in message] + # message = "\n".join(message) message_class = "danger" return {"message": message, "class": message_class} @@ -353,6 +354,8 @@ def query_results( return {"message": message, "class": message_class} if source != "all": add_bool.append({"src": source}) + + date_query = False if set({"from_date", "to_date", "from_time", "to_time"}).issubset( query_params.keys() ): @@ -366,7 +369,31 @@ def query_results( } } } - add_top.append(range_query) + if not settings.DELAY_RESULTS: + add_top.append(range_query) + else: + date_query = True + + if settings.DELAY_RESULTS: + if date_query: + if not request.user.has_perm("bypass_delay"): + delay_as_ts = datetime.now() - timedelta(days=settings.DELAY_DURATION) + lt_as_ts = datetime.strptime( + range_query["range"]["ts"]["lt"], "%Y-%m-%dT%H:%MZ" + ) + if lt_as_ts > delay_as_ts: + range_query["range"]["ts"]["lt"] = f"now-{settings.DELAY_DURATION}d" + add_top.append(range_query) + else: + range_query = { + "range": { + "ts": { + # "gt": , + "lt": f"now-{settings.DELAY_DURATION}d", + } + } + } + add_top.append(range_query) if "sorting" in query_params: sorting = query_params["sorting"] if sorting not in ("asc", "desc", "none"): @@ -556,6 +583,9 @@ def query_results( } if query: context["query"] = query + if settings.DELAY_RESULTS: + if not request.user.has_perm("bypass_delay"): + context["delay"] = settings.DELAY_DURATION return context diff --git a/core/migrations/0008_alter_perms_options.py b/core/migrations/0008_alter_perms_options.py new file mode 100644 index 0000000..957de9d --- /dev/null +++ b/core/migrations/0008_alter_perms_options.py @@ -0,0 +1,17 @@ +# Generated by Django 4.0.6 on 2022-08-27 11:50 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0007_perms'), + ] + + operations = [ + migrations.AlterModelOptions( + name='perms', + options={'permissions': (('bypass_hashing', 'Can bypass field hashing'), ('bypass_blacklist', 'Can bypass the blacklist'), ('bypass_encryption', 'Can bypass field encryption'), ('bypass_obfuscation', 'Can bypass field obfuscation'), ('bypass_delay', 'Can bypass data delay'), ('post_irc', 'Can post to IRC'), ('post_discord', 'Can post to Discord'), ('query_search', 'Can search with query strings'), ('use_insights', 'Can use the Insights page'), ('index_int', 'Can use the internal index'), ('index_meta', 'Can use the meta index'))}, + ), + ] diff --git a/core/models.py b/core/models.py index 829dc4d..72528ce 100644 --- a/core/models.py +++ b/core/models.py @@ -111,6 +111,7 @@ class Perms(models.Model): ("bypass_blacklist", "Can bypass the blacklist"), ("bypass_encryption", "Can bypass field encryption"), ("bypass_obfuscation", "Can bypass field obfuscation"), + ("bypass_delay", "Can bypass data delay"), ("post_irc", "Can post to IRC"), ("post_discord", "Can post to Discord"), ("query_search", "Can search with query strings"), diff --git a/core/templates/ui/drilldown/table_results.html b/core/templates/ui/drilldown/table_results.html index 2ecf004..616c2d8 100644 --- a/core/templates/ui/drilldown/table_results.html +++ b/core/templates/ui/drilldown/table_results.html @@ -26,6 +26,11 @@ {% endif %} {% endif %} + {% if delay is not None %} +
delayed by {{ delay }} days
+