Implement obfuscation

app/local_settings.example.py

@@ -24,6 +24,18 @@ ENCRYPTION_KEY = b""
 HASHING = True
 HASHING_KEY = "xxx"
 
+# Obfuscation
+OBFUSCATION = True
+# Fields obfuscate based on separators
+OBFUSCATE_FIELDS_SEP = ["date", "time"]
+# Fields to obfuscate based on length
+OBFUSCATE_FIELDS = ["ts"]
+OBFUSCATE_KEEP_RATIO = 0.9
+# DON'T obfuscate the last X fields of values separates by dashes
+OBFUSCATE_DASH_NUM = 2
+# DON'T obfuscate the last X fields of values separates by colons
+OBFUSCATE_COLON_NUM = 1
+
 # Common to encryption and hashing
 WHITELIST_FIELDS = [
     "ts",

core/lib/opensearch.py

@@ -11,6 +11,7 @@ from core.views.helpers import (
     encrypt_list,
     hash_list,
     hash_lookup,
+    obfuscate_list,
 )
 
 # from json import dumps
@@ -139,7 +140,7 @@ def filter_blacklisted(user, response):
                                 # Just set it to none so the index is not off
                                 response["hits"]["hits"][index] = None
                             else:
-                                if not user.is_superuser:
+                                if not user.has_perm("core.bypass_blacklist"):
                                     response["hits"]["hits"][index] = None
                                 else:
                                     response["hits"]["hits"][index][data_index][
@@ -526,9 +527,12 @@ def query_results(
     if settings.ENCRYPTION:
         encrypt_list(request.user, results_parsed, settings.ENCRYPTION_KEY)
 
-    if not request.user.has_perm("view_plain"):
-        if settings.HASHING:
-            hash_list(request.user, results_parsed)
+    if settings.HASHING:
+        hash_list(request.user, results_parsed)
+
+    if settings.OBFUSCATION:
+        obfuscate_list(request.user, results_parsed)
+
     # process_list(reqults)
 
     # IMPORTANT! - DO NOT PASS query_params to the user!

core/models.py

@@ -110,6 +110,7 @@ class Perms(models.Model):
             ("bypass_hashing", "Can bypass field hashing"),
             ("bypass_blacklist", "Can bypass the blacklist"),
             ("bypass_encryption", "Can bypass field encryption"),
+            ("bypass_obfuscation", "Can bypass field obfuscation"),
             ("post_irc", "Can post to IRC"),
             ("post_discord", "Can post to Discord"),
             ("query_search", "Can search with query strings"),

core/views/helpers.py

@@ -80,6 +80,45 @@ def base36decode(number):
     return int(number, 36)
 
 
+def obfuscate_list(user, data):
+    """
+    Obfuscate data in a list of dictionaries.
+    """
+    if user.has_perm("core.bypass_obfuscation"):
+        return
+    for index, item in enumerate(data):
+        for key, value in item.items():
+            # Obfuscate a ratio of the field
+            if key in settings.OBFUSCATE_FIELDS:
+                length = len(value) - 1
+                split = int(length * settings.OBFUSCATE_KEEP_RATIO)
+                first_part = value[:split]
+                second_part = value[split:]
+                second_len = len(second_part)
+                second_part = "*" * second_len
+                data[index][key] = first_part + second_part
+            # Obfuscate value based on fields
+            # Example: 2022-02-02 -> 2022-02-**
+            # 14:11:12 -> 14:11:**
+            elif key in settings.OBFUSCATE_FIELDS_SEP:
+                if "-" in value:
+                    sep = "-"
+                    value_spl = value.split("-")
+                    hide_num = settings.OBFUSCATE_DASH_NUM
+                elif ":" in value:
+                    sep = ":"
+                    value_spl = value.split(":")
+                    hide_num = settings.OBFUSCATE_COLON_NUM
+
+                first_part = value_spl[:hide_num]
+                second_part = value_spl[hide_num:]
+                for index_x, x in enumerate(second_part):
+                    x_len = len(x)
+                    second_part[index_x] = "*" * x_len
+                result = sep.join([*first_part, *second_part])
+                data[index][key] = result
+
+
 def hash_list(user, data, hash_keys=False):
     """
     Hash a list of dicts or a list with SipHash42.