Implement hashing bypass for groups

core/lib/context.py

@@ -23,6 +23,7 @@ def construct_query(index, net, channel, src, num, size, type=None, nicks=None):
         "type",
         "net",
         "src",
+        "tokens",
     ]
     if index == "int":
         fields.append("mtype")

core/lib/opensearch.py

@@ -448,10 +448,11 @@ def query_results(
         results_parsed = dedup_list(results_parsed, dedup_fields)
 
     if settings.ENCRYPTION:
-        encrypt_list(results_parsed, settings.ENCRYPTION_KEY)
+        encrypt_list(request.user, results_parsed, settings.ENCRYPTION_KEY)
 
+    if not request.user.has_perm("view_plain"):
         if settings.HASHING:
-        hash_list(results_parsed)
+            hash_list(request.user, results_parsed)
 
     # process_list(reqults)
 

core/migrations/0007_perms.py

@@ -0,0 +1,22 @@
+# Generated by Django 4.0.6 on 2022-08-16 18:04
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0006_contentblock_page'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Perms',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+            ],
+            options={
+                'permissions': (('bypass_hashing', 'Can bypass field hashing'), ('bypass_blacklist', 'Can bypass the blacklist'), ('bypass_encryption', 'Can bypass field encryption'), ('post_irc', 'Can post to IRC'), ('post_discord', 'Can post to Discord')),
+            },
+        ),
+    ]

core/models.py

@@ -104,15 +104,12 @@ class ContentBlock(models.Model):
         super().save(*args, **kwargs)
 
 
-class Role(models.Model):
+class Perms(models.Model):
-    name = models.CharField(max_length=255, unique=True)
+    class Meta:
-    description = models.CharField(max_length=1024, null=True, blank=True)
+        permissions = (
-    permission = models.CharField(max_length=255)
+            ("bypass_hashing", "Can bypass field hashing"),
-
+            ("bypass_blacklist", "Can bypass the blacklist"),
-    def __str__(self):
+            ("bypass_encryption", "Can bypass field encryption"),
-        return self.name
+            ("post_irc", "Can post to IRC"),
-
+            ("post_discord", "Can post to Discord"),
-
+        )
-class ContentPermission(models.Model):
-    inherit = models.ForeignKey("self", null=True, blank=True, on_delete=models.PROTECT)
-    roles = models.ManyToManyField(Role, blank=True)

core/templates/modals/context_table.html

@@ -82,6 +82,10 @@
                       <span class="icon" data-tooltip="Who">
                         <i class="fa-solid fa-passport"></i>
                       </span>
+                    {% elif item.type == 'topic' %}
+                      <span class="icon" data-tooltip="Topic">
+                        <i class="fa-solid fa-sign"></i>
+                      </span>
                     {% else %}
                       {{ item.type }}
                     {% endif %}

core/templates/ui/drilldown/drilldown.html

@@ -49,15 +49,9 @@
         populateSearch(field, value);
       });
     }
-    var plain_fields = ["ts", "date", "time", "sentiment", "version_sentiment", "tokens", "num_chans", "num_users", "tokens", "src", "exemption", "hidden"];
     function populateSearch(field, value) {
       var queryElement = document.getElementById('query');
 
-      if (!plain_fields.includes(field)) {
-        if (!value.startsWith("|") && !value.endsWith("|")) {
-          value = `|${value}|`;
-        }
-      }
       var present = true;
       if (present == true) {
         var combinations = [`${field}: "${value}"`,

core/templates/ui/drilldown/table_results_partial.html

@@ -122,6 +122,7 @@
                       <td>
                         <p class="has-text-grey">Hidden {{ row.cells.hidden }} similar result{% if row.cells.hidden > 1%}s{% endif %}</p>
                       </td>
+
                     </tr>
                   {% else %}
                     <tr class="
@@ -227,6 +228,10 @@
                                     <span class="icon" data-tooltip="Who">
                                       <i class="fa-solid fa-passport"></i>
                                     </span>
+                                  {% elif cell == 'topic' %}
+                                    <span class="icon" data-tooltip="Topic">
+                                      <i class="fa-solid fa-sign"></i>
+                                    </span>
                                   {% else %}
                                     {{ cell }}
                                   {% endif %}
@@ -238,16 +243,16 @@
                                   class="has-text-grey is-underlined"
                                   hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
                                   hx-post="{% url 'modal_context' %}"
-                                  hx-vals='{"net": "|{{ row.cells.net|escapejs }}|",
+                                  hx-vals='{"net": "{{ row.cells.net|escapejs }}",
-                                           "num": "|{{ row.cells.num|escapejs }}|",
+                                           "num": "{{ row.cells.num|escapejs }}",
                                            "src": "{{ row.cells.src|escapejs }}",
-                                           "channel": "|{{ row.cells.channel|escapejs }}|",
+                                           "channel": "{{ row.cells.channel|escapejs }}",
                                            "time": "{{ row.cells.time|escapejs }}",
                                            "date": "{{ row.cells.date|escapejs }}",
                                            "index": "{{ params.index }}",
-                                           "type": "|{{ row.cells.type }}|",
+                                           "type": "{{ row.cells.type }}",
                                            "mtype": "{{ row.cells.mtype }}",
-                                           "nick": "|{{ row.cells.nick|escapejs }}|",
+                                           "nick": "{{ row.cells.nick|escapejs }}",
                                            "dedup": "{{ params.dedup }}"}'
                                   hx-target="#modals-here" 
                                   hx-trigger="click"
@@ -281,7 +286,7 @@
                                       <button
                                         hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
                                         hx-post="{% url 'modal_drilldown' %}"
-                                        hx-vals='{"net": "|{{ row.cells.net }}|", "nick": "|{{ row.cells.nick }}|", "channel": "|{{ row.cells.channel }}|"}'
+                                        hx-vals='{"net": "{{ row.cells.net }}", "nick": "{{ row.cells.nick }}", "channel": "{{ row.cells.channel }}"}'
                                         hx-target="#modals-here" 
                                         hx-trigger="click"
                                         class="button is-small">

core/views/helpers.py

@@ -75,10 +75,12 @@ def base36decode(number):
     return int(number, 36)
 
 
-def hash_list(data, hash_keys=False):
+def hash_list(user, data, hash_keys=False):
     """
     Hash a list of dicts or a list with SipHash42.
     """
+    if user.has_perm("core.bypass_hashing"):
+        return
     cache = "cache.hash"
     hash_table = {}
     if isinstance(data, dict):
@@ -126,7 +128,8 @@ def hash_lookup(data_dict):
     for key, value in data_dict.items():
         if not value:
             continue
-        hashes = re.findall("\|([^\|]*)\|", value)  # noqa
+        # hashes = re.findall("\|([^\|]*)\|", value)  # noqa
+        hashes = re.findall("[A-Z0-9]{12,13}", value)
         if not hashes:
             continue
         for hash in hashes:
@@ -137,20 +140,20 @@ def hash_lookup(data_dict):
         if not values:
             return
         for index, val in enumerate(values):
-            if not val:
+            if val is None:
-                values[index] = "ERR"
+                values[index] = b"ERR"
         values = [x.decode() for x in values]
         total = dict(zip(hash_list, values))
         for key in data_dict.keys():
             for hash in total:
                 if data_dict[key]:
                     if hash in data_dict[key]:
-                        data_dict[key] = data_dict[key].replace(
+                        data_dict[key] = data_dict[key].replace(f"{hash}", total[hash])
-                            f"|{hash}|", total[hash]
-                        )
 
 
-def encrypt_list(data, secret):
+def encrypt_list(user, data, secret):
+    if user.has_perm("core.bypass_encryption"):
+        return
     cipher = Cipher(algorithms.AES(secret), ECB())
     for index, item in enumerate(data):
         for key, value in item.items():

core/views/ui/drilldown.py

@@ -106,7 +106,7 @@ def drilldown_search(request, return_context=False, template=None):
     query_params.update(tmp_get)
 
     if "index" in query_params:
-        if not request.user.is_superuser:
+        if not request.user.is_superuser and not query_params["index"] == "main":
             message = "You can't use the index parameter"
             message_class = "danger"
             context = {"message": message, "class": message_class}
@@ -253,7 +253,7 @@ class DrilldownContextModal(APIView):
             self.template_name = "modals/context_table.html"
 
         size = 20
-        nicks = None
+        nicks_sensitive = None
         query = False
         # Create the query params from the POST arguments
         mandatory = ["net", "channel", "num", "src", "index", "nick", "type", "mtype"]
@@ -272,51 +272,57 @@ class DrilldownContextModal(APIView):
         if settings.HASHING:
             SAFE_PARAMS = deepcopy(query_params)
             hash_lookup(SAFE_PARAMS)
+        else:
+            SAFE_PARAMS = query_params
 
         type = None
-        # SUPERUSER BLOCK #
         if request.user.is_superuser:
-            if "type" in SAFE_PARAMS:
+            if "type" in query_params:
-                type = SAFE_PARAMS["type"]
+                type = query_params["type"]
                 if type == "znc":
+                    query_params["channel"] = "*status"
                     SAFE_PARAMS["channel"] = "*status"
 
             if type in ["query", "notice"]:
-                nicks = [SAFE_PARAMS["channel"], SAFE_PARAMS["nick"]]
+                nicks_sensitive = [
+                    SAFE_PARAMS["channel"],
+                    SAFE_PARAMS["nick"],
+                ]  # UNSAFE
+                # nicks = [query_params["channel"], query_params["nick"]]
                 query = True
 
             if (
-                SAFE_PARAMS["index"] == "int"
+                query_params["index"] == "int"
-                and SAFE_PARAMS["mtype"] == "msg"
+                and query_params["mtype"] == "msg"
                 and not type == "query"
             ):
+                query_params["index"] = "main"
                 SAFE_PARAMS["index"] = "main"
 
-            if SAFE_PARAMS["type"] in ["znc", "auth"]:
+            if query_params["type"] in ["znc", "auth"]:
                 query = True
 
-        # SUPERUSER BLOCK #
-
         if not request.user.is_superuser:
-            if "index" in SAFE_PARAMS:
+            query_params["index"] = "main"
             SAFE_PARAMS["index"] = "main"
 
-        SAFE_PARAMS["sorting"] = "desc"
+        query_params["sorting"] = "desc"
+        SAFE_PARAMS["index"] = "main"
 
         annotate = False
-        if SAFE_PARAMS["src"] == "irc":
+        if query_params["src"] == "irc":
-            if SAFE_PARAMS["type"] in ["query", "notice", "msg", "highlight"]:
+            if query_params["type"] not in ["znc", "auth"]:
                 annotate = True
         # Create the query with the context helper
         search_query = construct_query(
-            SAFE_PARAMS["index"],
+            query_params["index"],
             SAFE_PARAMS["net"],
             SAFE_PARAMS["channel"],
-            SAFE_PARAMS["src"],
+            query_params["src"],
             SAFE_PARAMS["num"],
             size,
             type=type,
-            nicks=nicks,
+            nicks=nicks_sensitive,
         )
 
         results = query_results(
@@ -331,10 +337,18 @@ class DrilldownContextModal(APIView):
         if "message" in results:
             return render(request, self.template_name, results)
 
+        if settings.HASHING:  # we probably want to see the tokens
+            if not request.user.has_perm("bypass_hashing"):
+                for index, item in enumerate(results["object_list"]):
+                    if "tokens" in item:
+                        results["object_list"][index]["msg"] = results["object_list"][
+                            index
+                        ].pop("tokens")
+                        # item["msg"] = item.pop("tokens")
+
         # Make the time nicer
         # for index, item in enumerate(results["object_list"]):
         #    results["object_list"][index]["time"] = item["time"]+"SSS"
-
         context = {
             "net": query_params["net"],
             "channel": query_params["channel"],
@@ -396,18 +410,17 @@ class ThresholdInfoModal(APIView):
             inter_chans = get_chans(safe_net, users)
         else:
             inter_chans = []
-        hash_list(inter_chans)
+        hash_list(request.user, inter_chans)
-        hash_list(inter_users)
+        hash_list(request.user, inter_users)
 
-        hash_list(num_chans, hash_keys=True)
+        hash_list(request.user, num_chans, hash_keys=True)
-        hash_list(num_users, hash_keys=True)
+        hash_list(request.user, num_users, hash_keys=True)
 
-        hash_list(channels)
+        hash_list(request.user, channels)
-        hash_list(users)
+        hash_list(request.user, users)
 
         # SAFE BLOCK END #
-        nick = nick.replace("|", "")
+
-        channel = channel.replace("|", "")
         context = {
             "net": net,
             "nick": nick,

docker-compose.yml

@@ -14,6 +14,16 @@ services:
       - .env
     volumes_from:
         - tmp
+    depends_on:
+        - migration
+
+  migration:
+    image: pathogen/neptune:latest
+    command: sh -c '. /venv/bin/activate && python manage.py migrate --noinput'
+    volumes:
+      - ${PORTAINER_GIT_DIR}:/code
+      - ${NEPTUNE_LOCAL_SETTINGS}:/code/app/local_settings.py
+      - ${NEPTUNE_DATABASE_FILE}:/code/db.sqlite3
 
   # pyroscope:
   #   image: pyroscope/pyroscope

docker/docker-compose.prod.yml

@@ -15,6 +15,16 @@ services:
       - ../stack.env
     volumes_from:
         - tmp
+      depends_on:
+        - migration
+
+  migration:
+    image: pathogen/neptune:latest
+    command: sh -c '. /venv/bin/activate && python manage.py migrate --noinput'
+    volumes:
+      - ${PORTAINER_GIT_DIR}:/code
+      - ${NEPTUNE_LOCAL_SETTINGS}:/code/app/local_settings.py
+      - ${NEPTUNE_DATABASE_FILE}:/code/db.sqlite3
 
   tmp:
     image: busybox