From a2d572baf4a31a626aa56b35dd37dbb484ef58a3 Mon Sep 17 00:00:00 2001
From: Mark Veidemanis <m@zm.is>
Date: Sat, 27 Aug 2022 12:53:37 +0100
Subject: [PATCH] Delay results

---
 app/local_settings.example.py                 |  6 +++-
 core/lib/opensearch.py                        | 36 +++++++++++++++++--
 core/migrations/0008_alter_perms_options.py   | 17 +++++++++
 core/models.py                                |  1 +
 .../templates/ui/drilldown/table_results.html |  5 +++
 5 files changed, 61 insertions(+), 4 deletions(-)
 create mode 100644 core/migrations/0008_alter_perms_options.py

diff --git a/app/local_settings.example.py b/app/local_settings.example.py
index 6e2169e..e330b23 100644
--- a/app/local_settings.example.py
+++ b/app/local_settings.example.py
@@ -38,6 +38,10 @@ OBFUSCATE_COLON_NUM = 1
 
 SEARCH_FIELDS_DENY = ["ts", "date", "time"]
 
+DELAY_RESULTS = True
+# Delay results by this many days
+DELAY_DURATION = 10
+
 # Common to encryption and hashing
 WHITELIST_FIELDS = [
     "ts",
@@ -58,7 +62,7 @@ WHITELIST_FIELDS = [
 # Don't obfuscate these parameters, or lookup hashes in them
 NO_OBFUSCATE_PARAMS = [
     "query",
-#    "query_full",
+    #    "query_full",
     "size",
     "source",
     "sorting",
diff --git a/core/lib/opensearch.py b/core/lib/opensearch.py
index ccee8eb..3a3be8e 100644
--- a/core/lib/opensearch.py
+++ b/core/lib/opensearch.py
@@ -1,4 +1,5 @@
 from copy import deepcopy
+from datetime import datetime, timedelta
 
 from django.conf import settings
 from opensearchpy import OpenSearch
@@ -327,8 +328,8 @@ def query_results(
         elif isinstance(x, LookupDenied):
             message += f"Lookup({x.key}: {x.value}) "
     if denied:
-        #message = [f"{i}" for i in message]
-        #message = "\n".join(message)
+        # message = [f"{i}" for i in message]
+        # message = "\n".join(message)
         message_class = "danger"
         return {"message": message, "class": message_class}
 
@@ -353,6 +354,8 @@ def query_results(
             return {"message": message, "class": message_class}
         if source != "all":
             add_bool.append({"src": source})
+
+    date_query = False
     if set({"from_date", "to_date", "from_time", "to_time"}).issubset(
         query_params.keys()
     ):
@@ -366,7 +369,31 @@ def query_results(
                 }
             }
         }
-        add_top.append(range_query)
+        if not settings.DELAY_RESULTS:
+            add_top.append(range_query)
+        else:
+            date_query = True
+
+    if settings.DELAY_RESULTS:
+        if date_query:
+            if not request.user.has_perm("bypass_delay"):
+                delay_as_ts = datetime.now() - timedelta(days=settings.DELAY_DURATION)
+                lt_as_ts = datetime.strptime(
+                    range_query["range"]["ts"]["lt"], "%Y-%m-%dT%H:%MZ"
+                )
+                if lt_as_ts > delay_as_ts:
+                    range_query["range"]["ts"]["lt"] = f"now-{settings.DELAY_DURATION}d"
+                add_top.append(range_query)
+        else:
+            range_query = {
+                "range": {
+                    "ts": {
+                        # "gt": ,
+                        "lt": f"now-{settings.DELAY_DURATION}d",
+                    }
+                }
+            }
+            add_top.append(range_query)
     if "sorting" in query_params:
         sorting = query_params["sorting"]
         if sorting not in ("asc", "desc", "none"):
@@ -556,6 +583,9 @@ def query_results(
     }
     if query:
         context["query"] = query
+    if settings.DELAY_RESULTS:
+        if not request.user.has_perm("bypass_delay"):
+            context["delay"] = settings.DELAY_DURATION
     return context
 
 
diff --git a/core/migrations/0008_alter_perms_options.py b/core/migrations/0008_alter_perms_options.py
new file mode 100644
index 0000000..957de9d
--- /dev/null
+++ b/core/migrations/0008_alter_perms_options.py
@@ -0,0 +1,17 @@
+# Generated by Django 4.0.6 on 2022-08-27 11:50
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0007_perms'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='perms',
+            options={'permissions': (('bypass_hashing', 'Can bypass field hashing'), ('bypass_blacklist', 'Can bypass the blacklist'), ('bypass_encryption', 'Can bypass field encryption'), ('bypass_obfuscation', 'Can bypass field obfuscation'), ('bypass_delay', 'Can bypass data delay'), ('post_irc', 'Can post to IRC'), ('post_discord', 'Can post to Discord'), ('query_search', 'Can search with query strings'), ('use_insights', 'Can use the Insights page'), ('index_int', 'Can use the internal index'), ('index_meta', 'Can use the meta index'))},
+        ),
+    ]
diff --git a/core/models.py b/core/models.py
index 829dc4d..72528ce 100644
--- a/core/models.py
+++ b/core/models.py
@@ -111,6 +111,7 @@ class Perms(models.Model):
             ("bypass_blacklist", "Can bypass the blacklist"),
             ("bypass_encryption", "Can bypass field encryption"),
             ("bypass_obfuscation", "Can bypass field obfuscation"),
+            ("bypass_delay", "Can bypass data delay"),
             ("post_irc", "Can post to IRC"),
             ("post_discord", "Can post to Discord"),
             ("query_search", "Can search with query strings"),
diff --git a/core/templates/ui/drilldown/table_results.html b/core/templates/ui/drilldown/table_results.html
index 2ecf004..616c2d8 100644
--- a/core/templates/ui/drilldown/table_results.html
+++ b/core/templates/ui/drilldown/table_results.html
@@ -26,6 +26,11 @@
         </div>
       {% endif %}
     {% endif %}
+    {% if delay is not None %}
+      <div class="nowrap-child">
+        <p>delayed by {{ delay }} days</p>
+      </div>
+    {% endif %}
   </div>
   {% if params.index != 'int' and params.index != 'meta' %}
     <div class="box">