Pathogen
/
neptune
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Implement hashing bypass for groups

This commit is contained in:
Mark Veidemanis 2022-08-16 19:43:55 +01:00
parent e67eee8cc8
commit e08a7677ef
Signed by: m
GPG Key ID: 5ACFCEED46C0904F
11 changed files with 123 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
core/lib/context.py
View File

@ -23,6 +23,7 @@ def construct_query(index, net, channel, src, num, size, type=None, nicks=None):
"type",
"net",
"src",
"tokens",
]
if index == "int":
fields.append("mtype")

7
core/lib/opensearch.py
View File

@ -448,10 +448,11 @@ def query_results(
results_parsed = dedup_list(results_parsed, dedup_fields)
if settings.ENCRYPTION:
encrypt_list(results_parsed, settings.ENCRYPTION_KEY)
encrypt_list(request.user, results_parsed, settings.ENCRYPTION_KEY)
if settings.HASHING:
hash_list(results_parsed)
if not request.user.has_perm("view_plain"):
if settings.HASHING:
hash_list(request.user, results_parsed)
# process_list(reqults)

22
core/migrations/0007_perms.py Normal file
View File

@ -0,0 +1,22 @@
# Generated by Django 4.0.6 on 2022-08-16 18:04
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('core', '0006_contentblock_page'),
]
operations = [
migrations.CreateModel(
name='Perms',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
],
options={
'permissions': (('bypass_hashing', 'Can bypass field hashing'), ('bypass_blacklist', 'Can bypass the blacklist'), ('bypass_encryption', 'Can bypass field encryption'), ('post_irc', 'Can post to IRC'), ('post_discord', 'Can post to Discord')),
},
),
]

21
core/models.py
View File

@ -104,15 +104,12 @@ class ContentBlock(models.Model):
super().save(*args, **kwargs)
class Role(models.Model):
name = models.CharField(max_length=255, unique=True)
description = models.CharField(max_length=1024, null=True, blank=True)
permission = models.CharField(max_length=255)
def __str__(self):
return self.name
class ContentPermission(models.Model):
inherit = models.ForeignKey("self", null=True, blank=True, on_delete=models.PROTECT)
roles = models.ManyToManyField(Role, blank=True)
class Perms(models.Model):
class Meta:
permissions = (
("bypass_hashing", "Can bypass field hashing"),
("bypass_blacklist", "Can bypass the blacklist"),
("bypass_encryption", "Can bypass field encryption"),
("post_irc", "Can post to IRC"),
("post_discord", "Can post to Discord"),
)

4
core/templates/modals/context_table.html
View File

@ -82,6 +82,10 @@
<span class="icon" data-tooltip="Who">
<i class="fa-solid fa-passport"></i>
</span>
{% elif item.type == 'topic' %}
<span class="icon" data-tooltip="Topic">
<i class="fa-solid fa-sign"></i>
</span>
{% else %}
{{ item.type }}
{% endif %}

6
core/templates/ui/drilldown/drilldown.html
View File

@ -49,15 +49,9 @@
populateSearch(field, value);
});
}
var plain_fields = ["ts", "date", "time", "sentiment", "version_sentiment", "tokens", "num_chans", "num_users", "tokens", "src", "exemption", "hidden"];
function populateSearch(field, value) {
var queryElement = document.getElementById('query');
if (!plain_fields.includes(field)) {
if (!value.startsWith("|") && !value.endsWith("|")) {
value = `|${value}|`;
}
}
var present = true;
if (present == true) {
var combinations = [`${field}: "${value}"`,

17
core/templates/ui/drilldown/table_results_partial.html
View File

@ -122,6 +122,7 @@
<td>
<p class="has-text-grey">Hidden {{ row.cells.hidden }} similar result{% if row.cells.hidden > 1%}s{% endif %}</p>
</td>
</tr>
{% else %}
<tr class="
@ -227,6 +228,10 @@
<span class="icon" data-tooltip="Who">
<i class="fa-solid fa-passport"></i>
</span>
{% elif cell == 'topic' %}
<span class="icon" data-tooltip="Topic">
<i class="fa-solid fa-sign"></i>
</span>
{% else %}
{{ cell }}
{% endif %}
@ -238,16 +243,16 @@
class="has-text-grey is-underlined"
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
hx-post="{% url 'modal_context' %}"
hx-vals='{"net": "|{{ row.cells.net|escapejs }}|",
"num": "|{{ row.cells.num|escapejs }}|",
hx-vals='{"net": "{{ row.cells.net|escapejs }}",
"num": "{{ row.cells.num|escapejs }}",
"src": "{{ row.cells.src|escapejs }}",
"channel": "|{{ row.cells.channel|escapejs }}|",
"channel": "{{ row.cells.channel|escapejs }}",
"time": "{{ row.cells.time|escapejs }}",
"date": "{{ row.cells.date|escapejs }}",
"index": "{{ params.index }}",
"type": "|{{ row.cells.type }}|",
"type": "{{ row.cells.type }}",
"mtype": "{{ row.cells.mtype }}",
"nick": "|{{ row.cells.nick|escapejs }}|",
"nick": "{{ row.cells.nick|escapejs }}",
"dedup": "{{ params.dedup }}"}'
hx-target="#modals-here"
hx-trigger="click"
@ -281,7 +286,7 @@
<button
hx-headers='{"X-CSRFToken": "{{ csrf_token }}"}'
hx-post="{% url 'modal_drilldown' %}"
hx-vals='{"net": "|{{ row.cells.net }}|", "nick": "|{{ row.cells.nick }}|", "channel": "|{{ row.cells.channel }}|"}'
hx-vals='{"net": "{{ row.cells.net }}", "nick": "{{ row.cells.nick }}", "channel": "{{ row.cells.channel }}"}'
hx-target="#modals-here"
hx-trigger="click"
class="button is-small">

19
core/views/helpers.py
View File

@ -75,10 +75,12 @@ def base36decode(number):
return int(number, 36)
def hash_list(data, hash_keys=False):
def hash_list(user, data, hash_keys=False):
"""
Hash a list of dicts or a list with SipHash42.
"""
if user.has_perm("core.bypass_hashing"):
return
cache = "cache.hash"
hash_table = {}
if isinstance(data, dict):
@ -126,7 +128,8 @@ def hash_lookup(data_dict):
for key, value in data_dict.items():
if not value:
continue
hashes = re.findall("\|([^\|]*)\|", value) # noqa
# hashes = re.findall("\|([^\|]*)\|", value) # noqa
hashes = re.findall("[A-Z0-9]{12,13}", value)
if not hashes:
continue
for hash in hashes:
@ -137,20 +140,20 @@ def hash_lookup(data_dict):
if not values:
return
for index, val in enumerate(values):
if not val:
values[index] = "ERR"
if val is None:
values[index] = b"ERR"
values = [x.decode() for x in values]
total = dict(zip(hash_list, values))
for key in data_dict.keys():
for hash in total:
if data_dict[key]:
if hash in data_dict[key]:
data_dict[key] = data_dict[key].replace(
f"|{hash}|", total[hash]
)
data_dict[key] = data_dict[key].replace(f"{hash}", total[hash])
def encrypt_list(data, secret):
def encrypt_list(user, data, secret):
if user.has_perm("core.bypass_encryption"):
return
cipher = Cipher(algorithms.AES(secret), ECB())
for index, item in enumerate(data):
for key, value in item.items():

69
core/views/ui/drilldown.py
View File

@ -106,7 +106,7 @@ def drilldown_search(request, return_context=False, template=None):
query_params.update(tmp_get)
if "index" in query_params:
if not request.user.is_superuser:
if not request.user.is_superuser and not query_params["index"] == "main":
message = "You can't use the index parameter"
message_class = "danger"
context = {"message": message, "class": message_class}
@ -253,7 +253,7 @@ class DrilldownContextModal(APIView):
self.template_name = "modals/context_table.html"
size = 20
nicks = None
nicks_sensitive = None
query = False
# Create the query params from the POST arguments
mandatory = ["net", "channel", "num", "src", "index", "nick", "type", "mtype"]
@ -272,51 +272,57 @@ class DrilldownContextModal(APIView):
if settings.HASHING:
SAFE_PARAMS = deepcopy(query_params)
hash_lookup(SAFE_PARAMS)
else:
SAFE_PARAMS = query_params
type = None
# SUPERUSER BLOCK #
if request.user.is_superuser:
if "type" in SAFE_PARAMS:
type = SAFE_PARAMS["type"]
if "type" in query_params:
type = query_params["type"]
if type == "znc":
query_params["channel"] = "*status"
SAFE_PARAMS["channel"] = "*status"
if type in ["query", "notice"]:
nicks = [SAFE_PARAMS["channel"], SAFE_PARAMS["nick"]]
nicks_sensitive = [
SAFE_PARAMS["channel"],
SAFE_PARAMS["nick"],
] # UNSAFE
# nicks = [query_params["channel"], query_params["nick"]]
query = True
if (
SAFE_PARAMS["index"] == "int"
and SAFE_PARAMS["mtype"] == "msg"
query_params["index"] == "int"
and query_params["mtype"] == "msg"
and not type == "query"
):
query_params["index"] = "main"
SAFE_PARAMS["index"] = "main"
if SAFE_PARAMS["type"] in ["znc", "auth"]:
if query_params["type"] in ["znc", "auth"]:
query = True
# SUPERUSER BLOCK #
if not request.user.is_superuser:
if "index" in SAFE_PARAMS:
SAFE_PARAMS["index"] = "main"
query_params["index"] = "main"
SAFE_PARAMS["index"] = "main"
SAFE_PARAMS["sorting"] = "desc"
query_params["sorting"] = "desc"
SAFE_PARAMS["index"] = "main"
annotate = False
if SAFE_PARAMS["src"] == "irc":
if SAFE_PARAMS["type"] in ["query", "notice", "msg", "highlight"]:
if query_params["src"] == "irc":
if query_params["type"] not in ["znc", "auth"]:
annotate = True
# Create the query with the context helper
search_query = construct_query(
SAFE_PARAMS["index"],
query_params["index"],
SAFE_PARAMS["net"],
SAFE_PARAMS["channel"],
SAFE_PARAMS["src"],
query_params["src"],
SAFE_PARAMS["num"],
size,
type=type,
nicks=nicks,
nicks=nicks_sensitive,
)
results = query_results(
@ -331,10 +337,18 @@ class DrilldownContextModal(APIView):
if "message" in results:
return render(request, self.template_name, results)
if settings.HASHING: # we probably want to see the tokens
if not request.user.has_perm("bypass_hashing"):
for index, item in enumerate(results["object_list"]):
if "tokens" in item:
results["object_list"][index]["msg"] = results["object_list"][
index
].pop("tokens")
# item["msg"] = item.pop("tokens")
# Make the time nicer
# for index, item in enumerate(results["object_list"]):
# results["object_list"][index]["time"] = item["time"]+"SSS"
context = {
"net": query_params["net"],
"channel": query_params["channel"],
@ -396,18 +410,17 @@ class ThresholdInfoModal(APIView):
inter_chans = get_chans(safe_net, users)
else:
inter_chans = []
hash_list(inter_chans)
hash_list(inter_users)
hash_list(request.user, inter_chans)
hash_list(request.user, inter_users)
hash_list(num_chans, hash_keys=True)
hash_list(num_users, hash_keys=True)
hash_list(request.user, num_chans, hash_keys=True)
hash_list(request.user, num_users, hash_keys=True)
hash_list(channels)
hash_list(users)
hash_list(request.user, channels)
hash_list(request.user, users)
# SAFE BLOCK END #
nick = nick.replace("|", "")
channel = channel.replace("|", "")
context = {
"net": net,
"nick": nick,

10
docker-compose.yml
View File

@ -14,6 +14,16 @@ services:
- .env
volumes_from:
- tmp
depends_on:
- migration
migration:
image: pathogen/neptune:latest
command: sh -c '. /venv/bin/activate && python manage.py migrate --noinput'
volumes:
- ${PORTAINER_GIT_DIR}:/code
- ${NEPTUNE_LOCAL_SETTINGS}:/code/app/local_settings.py
- ${NEPTUNE_DATABASE_FILE}:/code/db.sqlite3
# pyroscope:
# image: pyroscope/pyroscope

10
docker/docker-compose.prod.yml
View File

@ -15,6 +15,16 @@ services:
- ../stack.env
volumes_from:
- tmp
depends_on:
- migration
migration:
image: pathogen/neptune:latest
command: sh -c '. /venv/bin/activate && python manage.py migrate --noinput'
volumes:
- ${PORTAINER_GIT_DIR}:/code
- ${NEPTUNE_LOCAL_SETTINGS}:/code/app/local_settings.py
- ${NEPTUNE_DATABASE_FILE}:/code/db.sqlite3
tmp:
image: busybox