Delay results

app/local_settings.example.py

@@ -38,6 +38,10 @@ OBFUSCATE_COLON_NUM = 1
 
 SEARCH_FIELDS_DENY = ["ts", "date", "time"]
 
+DELAY_RESULTS = True
+# Delay results by this many days
+DELAY_DURATION = 10
+
 # Common to encryption and hashing
 WHITELIST_FIELDS = [
     "ts",
@@ -58,7 +62,7 @@ WHITELIST_FIELDS = [
 # Don't obfuscate these parameters, or lookup hashes in them
 NO_OBFUSCATE_PARAMS = [
     "query",
-#    "query_full",
+    #    "query_full",
     "size",
     "source",
     "sorting",

core/lib/opensearch.py

@@ -1,4 +1,5 @@
 from copy import deepcopy
+from datetime import datetime, timedelta
 
 from django.conf import settings
 from opensearchpy import OpenSearch
@@ -327,8 +328,8 @@ def query_results(
         elif isinstance(x, LookupDenied):
             message += f"Lookup({x.key}: {x.value}) "
     if denied:
-        #message = [f"{i}" for i in message]
+        # message = [f"{i}" for i in message]
-        #message = "\n".join(message)
+        # message = "\n".join(message)
         message_class = "danger"
         return {"message": message, "class": message_class}
 
@@ -353,6 +354,8 @@ def query_results(
             return {"message": message, "class": message_class}
         if source != "all":
             add_bool.append({"src": source})
+
+    date_query = False
     if set({"from_date", "to_date", "from_time", "to_time"}).issubset(
         query_params.keys()
     ):
@@ -366,6 +369,30 @@ def query_results(
                 }
             }
         }
+        if not settings.DELAY_RESULTS:
+            add_top.append(range_query)
+        else:
+            date_query = True
+
+    if settings.DELAY_RESULTS:
+        if date_query:
+            if not request.user.has_perm("bypass_delay"):
+                delay_as_ts = datetime.now() - timedelta(days=settings.DELAY_DURATION)
+                lt_as_ts = datetime.strptime(
+                    range_query["range"]["ts"]["lt"], "%Y-%m-%dT%H:%MZ"
+                )
+                if lt_as_ts > delay_as_ts:
+                    range_query["range"]["ts"]["lt"] = f"now-{settings.DELAY_DURATION}d"
+                add_top.append(range_query)
+        else:
+            range_query = {
+                "range": {
+                    "ts": {
+                        # "gt": ,
+                        "lt": f"now-{settings.DELAY_DURATION}d",
+                    }
+                }
+            }
             add_top.append(range_query)
     if "sorting" in query_params:
         sorting = query_params["sorting"]
@@ -556,6 +583,9 @@ def query_results(
     }
     if query:
         context["query"] = query
+    if settings.DELAY_RESULTS:
+        if not request.user.has_perm("bypass_delay"):
+            context["delay"] = settings.DELAY_DURATION
     return context
 
 

core/migrations/0008_alter_perms_options.py

@@ -0,0 +1,17 @@
+# Generated by Django 4.0.6 on 2022-08-27 11:50
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0007_perms'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='perms',
+            options={'permissions': (('bypass_hashing', 'Can bypass field hashing'), ('bypass_blacklist', 'Can bypass the blacklist'), ('bypass_encryption', 'Can bypass field encryption'), ('bypass_obfuscation', 'Can bypass field obfuscation'), ('bypass_delay', 'Can bypass data delay'), ('post_irc', 'Can post to IRC'), ('post_discord', 'Can post to Discord'), ('query_search', 'Can search with query strings'), ('use_insights', 'Can use the Insights page'), ('index_int', 'Can use the internal index'), ('index_meta', 'Can use the meta index'))},
+        ),
+    ]

core/models.py

@@ -111,6 +111,7 @@ class Perms(models.Model):
             ("bypass_blacklist", "Can bypass the blacklist"),
             ("bypass_encryption", "Can bypass field encryption"),
             ("bypass_obfuscation", "Can bypass field obfuscation"),
+            ("bypass_delay", "Can bypass data delay"),
             ("post_irc", "Can post to IRC"),
             ("post_discord", "Can post to Discord"),
             ("query_search", "Can search with query strings"),

core/templates/ui/drilldown/table_results.html

@@ -26,6 +26,11 @@
         </div>
       {% endif %}
     {% endif %}
+    {% if delay is not None %}
+      <div class="nowrap-child">
+        <p>delayed by {{ delay }} days</p>
+      </div>
+    {% endif %}
   </div>
   {% if params.index != 'int' and params.index != 'meta' %}
     <div class="box">