Pathogen
/
neptune
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Remove redaction stuff

This commit is contained in:
Mark Veidemanis 2022-08-26 07:20:30 +01:00
parent cc20c545dd
commit bdee5a2aae
Signed by: m
GPG Key ID: 5ACFCEED46C0904F
5 changed files with 404 additions and 396 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

204
core/lib/opensearch.py
View File

@ -1,21 +1,12 @@
from copy import deepcopy # from copy import deepcopy
from datetime import datetime, timedelta # from datetime import datetime, timedelta
from django.conf import settings from django.conf import settings
from opensearchpy import OpenSearch from opensearchpy import OpenSearch
from opensearchpy.exceptions import NotFoundError, RequestError from opensearchpy.exceptions import NotFoundError, RequestError
from core.lib.threshold import annotate_num_chans, annotate_num_users, annotate_online from core.lib.threshold import annotate_num_chans, annotate_num_users, annotate_online
from core.views.helpers import ( from core.views.helpers import dedup_list
LookupDenied,
SearchDenied,
dedup_list,
encrypt_list,
hash_list,
hash_lookup,
obfuscate_list,
randomise_list,
)
# from json import dumps # from json import dumps
# pp = lambda x: print(dumps(x, indent=2)) # pp = lambda x: print(dumps(x, indent=2))
@ -216,9 +207,7 @@ def construct_query(query, size, use_query_string=True, tokens=False):
return query_base return query_base
def run_main_query( def run_main_query(client, user, query, custom_query=False, index=None, size=None):
client, user, query, custom_query=False, index=None, size=None, filter=True
):
""" """
Low level helper to run an ES query. Low level helper to run an ES query.
Accept a user to pass it to the filter, so we can Accept a user to pass it to the filter, so we can
@ -240,7 +229,6 @@ def run_main_query(
except NotFoundError as err: except NotFoundError as err:
print("OpenSearch error", err) print("OpenSearch error", err)
return err return err
if filter:
filter_blacklisted(user, response) filter_blacklisted(user, response)
return response return response
@ -319,27 +307,27 @@ def query_results(
query_created = False query_created = False
# Lookup the hash values but don't disclose them to the user # Lookup the hash values but don't disclose them to the user
denied = [] # denied = []
if lookup_hashes: # if lookup_hashes:
if settings.HASHING: # if settings.HASHING:
query_params = deepcopy(query_params) # query_params = deepcopy(query_params)
denied_q = hash_lookup(request.user, query_params) # denied_q = hash_lookup(request.user, query_params)
denied.extend(denied_q) # denied.extend(denied_q)
if tags: # if tags:
denied_t = hash_lookup(request.user, tags, query_params) # denied_t = hash_lookup(request.user, tags, query_params)
denied.extend(denied_t) # denied.extend(denied_t)
message = "Permission denied: " # message = "Permission denied: "
for x in denied: # for x in denied:
if isinstance(x, SearchDenied): # if isinstance(x, SearchDenied):
message += f"Search({x.key}: {x.value}) " # message += f"Search({x.key}: {x.value}) "
elif isinstance(x, LookupDenied): # elif isinstance(x, LookupDenied):
message += f"Lookup({x.key}: {x.value}) " # message += f"Lookup({x.key}: {x.value}) "
if denied: # if denied:
# message = [f"{i}" for i in message] # # message = [f"{i}" for i in message]
# message = "\n".join(message) # # message = "\n".join(message)
message_class = "danger" # message_class = "danger"
return {"message": message, "class": message_class} # return {"message": message, "class": message_class}
if request.user.is_anonymous: if request.user.is_anonymous:
sizes = settings.OPENSEARCH_MAIN_SIZES_ANON sizes = settings.OPENSEARCH_MAIN_SIZES_ANON
@ -357,14 +345,36 @@ def query_results(
source = None source = None
if "source" in query_params: if "source" in query_params:
source = query_params["source"] source = query_params["source"]
if source not in settings.OPENSEARCH_MAIN_SOURCES:
if source in settings.OPENSEARCH_SOURCES_RESTRICTED:
if not request.user.has_perm("restricted_sources"):
message = "Access denied"
message_class = "danger"
return {"message": message, "class": message_class}
elif source not in settings.OPENSEARCH_MAIN_SOURCES:
message = "Invalid source" message = "Invalid source"
message_class = "danger" message_class = "danger"
return {"message": message, "class": message_class} return {"message": message, "class": message_class}
if source != "all":
add_bool.append({"src": source})
date_query = False if source == "all":
source = None # the next block will populate it
if source:
sources = [source]
else:
sources = settings.OPENSEARCH_MAIN_SOURCES
if request.user.has_perm("restricted_sources"):
for source_iter in settings.OPENSEARCH_SOURCES_RESTRICTED:
sources.append(source_iter)
else:
sources = settings.OPENSEARCH_MAIN_SOURCES
add_top_tmp = {"bool": {"should": []}}
for source_iter in sources:
add_top_tmp["bool"]["should"].append({"match_phrase": {"src": source_iter}})
add_top.append(add_top_tmp)
# date_query = False
if set({"from_date", "to_date", "from_time", "to_time"}).issubset( if set({"from_date", "to_date", "from_time", "to_time"}).issubset(
query_params.keys() query_params.keys()
): ):
@ -378,40 +388,40 @@ def query_results(
} }
} }
} }
date_query = True add_top.append(range_query)
if date_query: # if date_query:
if settings.DELAY_RESULTS: # if settings.DELAY_RESULTS:
if source not in settings.SAFE_SOURCES: # if source not in settings.SAFE_SOURCES:
if request.user.has_perm("core.bypass_delay"): # if request.user.has_perm("core.bypass_delay"):
add_top.append(range_query) # add_top.append(range_query)
else: # else:
delay_as_ts = datetime.now() - timedelta( # delay_as_ts = datetime.now() - timedelta(
days=settings.DELAY_DURATION # days=settings.DELAY_DURATION
) # )
lt_as_ts = datetime.strptime( # lt_as_ts = datetime.strptime(
range_query["range"]["ts"]["lt"], "%Y-%m-%dT%H:%MZ" # range_query["range"]["ts"]["lt"], "%Y-%m-%dT%H:%MZ"
) # )
if lt_as_ts > delay_as_ts: # if lt_as_ts > delay_as_ts:
range_query["range"]["ts"][ # range_query["range"]["ts"][
"lt" # "lt"
] = f"now-{settings.DELAY_DURATION}d" # ] = f"now-{settings.DELAY_DURATION}d"
add_top.append(range_query) # add_top.append(range_query)
else: # else:
add_top.append(range_query) # add_top.append(range_query)
else: # else:
if settings.DELAY_RESULTS: # if settings.DELAY_RESULTS:
if source not in settings.SAFE_SOURCES: # if source not in settings.SAFE_SOURCES:
if not request.user.has_perm("core.bypass_delay"): # if not request.user.has_perm("core.bypass_delay"):
range_query = { # range_query = {
"range": { # "range": {
"ts": { # "ts": {
# "gt": , # # "gt": ,
"lt": f"now-{settings.DELAY_DURATION}d", # "lt": f"now-{settings.DELAY_DURATION}d",
} # }
} # }
} # }
add_top.append(range_query) # add_top.append(range_query)
if "sorting" in query_params: if "sorting" in query_params:
sorting = query_params["sorting"] sorting = query_params["sorting"]
@ -469,13 +479,13 @@ def query_results(
query_created = True query_created = True
elif "query_full" in query_params: elif "query_full" in query_params:
query_full = query_params["query_full"] query_full = query_params["query_full"]
if request.user.has_perm("core.query_search"): # if request.user.has_perm("core.query_search"):
search_query = construct_query(query_full, size) search_query = construct_query(query_full, size)
query_created = True query_created = True
else: # else:
message = "You cannot search by query string" # message = "You cannot search by query string"
message_class = "danger" # message_class = "danger"
return {"message": message, "class": message_class} # return {"message": message, "class": message_class}
else: else:
if custom_query: if custom_query:
search_query = custom_query search_query = custom_query
@ -542,9 +552,6 @@ def query_results(
else: else:
index = settings.OPENSEARCH_INDEX_MAIN index = settings.OPENSEARCH_INDEX_MAIN
filter = True
if source in settings.SAFE_SOURCES:
filter = False
results = run_main_query( results = run_main_query(
client, client,
request.user, # passed through run_main_query to filter_blacklisted request.user, # passed through run_main_query to filter_blacklisted
@ -552,7 +559,6 @@ def query_results(
custom_query=True, custom_query=True,
index=index, index=index,
size=size, size=size,
filter=filter,
) )
if not results: if not results:
return False return False
@ -585,18 +591,18 @@ def query_results(
dedup_fields = ["msg", "nick", "ident", "host", "net", "channel"] dedup_fields = ["msg", "nick", "ident", "host", "net", "channel"]
results_parsed = dedup_list(results_parsed, dedup_fields) results_parsed = dedup_list(results_parsed, dedup_fields)
if source not in settings.SAFE_SOURCES: # if source not in settings.SAFE_SOURCES:
if settings.ENCRYPTION: # if settings.ENCRYPTION:
encrypt_list(request.user, results_parsed, settings.ENCRYPTION_KEY) # encrypt_list(request.user, results_parsed, settings.ENCRYPTION_KEY)
if settings.HASHING: # if settings.HASHING:
hash_list(request.user, results_parsed) # hash_list(request.user, results_parsed)
if settings.OBFUSCATION: # if settings.OBFUSCATION:
obfuscate_list(request.user, results_parsed) # obfuscate_list(request.user, results_parsed)
if settings.RANDOMISATION: # if settings.RANDOMISATION:
randomise_list(request.user, results_parsed) # randomise_list(request.user, results_parsed)
# process_list(results) # process_list(results)
@ -612,14 +618,14 @@ def query_results(
context["exemption"] = results["exemption"] context["exemption"] = results["exemption"]
if query: if query:
context["query"] = query context["query"] = query
if settings.DELAY_RESULTS: # if settings.DELAY_RESULTS:
if source not in settings.SAFE_SOURCES: # if source not in settings.SAFE_SOURCES:
if not request.user.has_perm("core.bypass_delay"): # if not request.user.has_perm("core.bypass_delay"):
context["delay"] = settings.DELAY_DURATION # context["delay"] = settings.DELAY_DURATION
if settings.RANDOMISATION: # if settings.RANDOMISATION:
if source not in settings.SAFE_SOURCES: # if source not in settings.SAFE_SOURCES:
if not request.user.has_perm("core.bypass_randomisation"): # if not request.user.has_perm("core.bypass_randomisation"):
context["randomised"] = True # context["randomised"] = True
return context return context

15
core/models.py
View File

@ -107,16 +107,17 @@ class ContentBlock(models.Model):
class Perms(models.Model): class Perms(models.Model):
class Meta: class Meta:
permissions = ( permissions = (
("bypass_hashing", "Can bypass field hashing"), ("bypass_hashing", "Can bypass field hashing"), #
("bypass_blacklist", "Can bypass the blacklist"), ("bypass_blacklist", "Can bypass the blacklist"), #
("bypass_encryption", "Can bypass field encryption"), ("bypass_encryption", "Can bypass field encryption"), #
("bypass_obfuscation", "Can bypass field obfuscation"), ("bypass_obfuscation", "Can bypass field obfuscation"), #
("bypass_delay", "Can bypass data delay"), ("bypass_delay", "Can bypass data delay"), #
("bypass_randomisation", "Can bypass data randomisation"), ("bypass_randomisation", "Can bypass data randomisation"), #
("post_irc", "Can post to IRC"), ("post_irc", "Can post to IRC"),
("post_discord", "Can post to Discord"), ("post_discord", "Can post to Discord"),
("query_search", "Can search with query strings"), ("query_search", "Can search with query strings"), #
("use_insights", "Can use the Insights page"), ("use_insights", "Can use the Insights page"),
("index_int", "Can use the internal index"), ("index_int", "Can use the internal index"),
("index_meta", "Can use the meta index"), ("index_meta", "Can use the meta index"),
("restricted_sources", "Can access restricted sources"),
) )

6
core/templates/ui/drilldown/search_partial.html
View File

@ -83,7 +83,8 @@
type="checkbox" type="checkbox"
class="switch is-rounded is-info" class="switch is-rounded is-info"
{% if params.query_full is not None %}checked="checked"{% else %}none{% endif %} {% if params.query_full is not None %}checked="checked"{% else %}none{% endif %}
{% if not perms.core.query_search %} {% if False %}
{# what are you looking at? #}
disabled disabled
{% endif %} {% endif %}
data-script="on click toggle .is-hidden on #query_full"> data-script="on click toggle .is-hidden on #query_full">
@ -91,7 +92,8 @@
for="full_query"> for="full_query">
Full query Full query
</label> </label>
{% if not perms.core.query_search %} {% if False %}
{# what are you looking at? #}
<span class="tooltiptext tag is-danger is-light">No access</span> <span class="tooltiptext tag is-danger is-light">No access</span>
{% endif %} {% endif %}
</div> </div>

469
core/views/helpers.py
View File

@ -1,14 +1,14 @@
import re # import re
from base64 import b64encode # from base64 import b64encode
from random import randint # from random import randint
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms # from cryptography.hazmat.primitives.ciphers import Cipher, algorithms
from cryptography.hazmat.primitives.ciphers.modes import ECB # from cryptography.hazmat.primitives.ciphers.modes import ECB
from django.conf import settings # from django.conf import settings
from siphashc import siphash # from siphashc import siphash
from sortedcontainers import SortedSet # from sortedcontainers import SortedSet
from core import r # from core import r
class SearchDenied: class SearchDenied:
@ -62,248 +62,249 @@ def dedup_list(data, check_keys):
# # 1.0805372429895215 # # 1.0805372429895215
def base36encode(number, alphabet="0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"): # def base36encode(number, alphabet="0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"):
"""Converts an integer to a base36 string.""" # """Converts an integer to a base36 string."""
if not isinstance(number, (int)): # if not isinstance(number, (int)):
raise TypeError("number must be an integer") # raise TypeError("number must be an integer")
base36 = "" # base36 = ""
sign = "" # sign = ""
if number < 0: # if number < 0:
sign = "-" # sign = "-"
number = -number # number = -number
if 0 <= number < len(alphabet): # if 0 <= number < len(alphabet):
return sign + alphabet[number] # return sign + alphabet[number]
while number != 0: # while number != 0:
number, i = divmod(number, len(alphabet)) # number, i = divmod(number, len(alphabet))
base36 = alphabet[i] + base36 # base36 = alphabet[i] + base36
return sign + base36 # return sign + base36
def base36decode(number): # def base36decode(number):
return int(number, 36) # return int(number, 36)
def randomise_list(user, data): # def randomise_list(user, data):
""" # """
Randomise data in a list of dictionaries. # Randomise data in a list of dictionaries.
""" # """
if user.has_perm("core.bypass_randomisation"): # if user.has_perm("core.bypass_randomisation"):
return # return
if isinstance(data, list): # if isinstance(data, list):
for index, item in enumerate(data): # for index, item in enumerate(data):
for key, value in item.items(): # for key, value in item.items():
if key in settings.RANDOMISE_FIELDS: # if key in settings.RANDOMISE_FIELDS:
if isinstance(value, int): # if isinstance(value, int):
min_val = value - (value * settings.RANDOMISE_RATIO) # min_val = value - (value * settings.RANDOMISE_RATIO)
max_val = value + (value * settings.RANDOMISE_RATIO) # max_val = value + (value * settings.RANDOMISE_RATIO)
new_val = randint(int(min_val), int(max_val)) # new_val = randint(int(min_val), int(max_val))
data[index][key] = new_val # data[index][key] = new_val
elif isinstance(data, dict): # elif isinstance(data, dict):
for key, value in data.items(): # for key, value in data.items():
# if key in settings.RANDOMISE_FIELDS: # # if key in settings.RANDOMISE_FIELDS:
if isinstance(value, int): # if isinstance(value, int):
min_val = value - (value * settings.RANDOMISE_RATIO) # min_val = value - (value * settings.RANDOMISE_RATIO)
max_val = value + (value * settings.RANDOMISE_RATIO) # max_val = value + (value * settings.RANDOMISE_RATIO)
new_val = randint(int(min_val), int(max_val)) # new_val = randint(int(min_val), int(max_val))
data[key] = new_val # data[key] = new_val
def obfuscate_list(user, data): # def obfuscate_list(user, data):
""" # """
Obfuscate data in a list of dictionaries. # Obfuscate data in a list of dictionaries.
""" # """
if user.has_perm("core.bypass_obfuscation"): # if user.has_perm("core.bypass_obfuscation"):
return # return
for index, item in enumerate(data): # for index, item in enumerate(data):
for key, value in item.items(): # for key, value in item.items():
# Obfuscate a ratio of the field # # Obfuscate a ratio of the field
if key in settings.OBFUSCATE_FIELDS: # if key in settings.OBFUSCATE_FIELDS:
length = len(value) - 1 # length = len(value) - 1
split = int(length * settings.OBFUSCATE_KEEP_RATIO) # split = int(length * settings.OBFUSCATE_KEEP_RATIO)
first_part = value[:split] # first_part = value[:split]
second_part = value[split:] # second_part = value[split:]
second_len = len(second_part) # second_len = len(second_part)
second_part = "*" * second_len # second_part = "*" * second_len
data[index][key] = first_part + second_part # data[index][key] = first_part + second_part
# Obfuscate value based on fields # # Obfuscate value based on fields
# Example: 2022-02-02 -> 2022-02-** # # Example: 2022-02-02 -> 2022-02-**
# 14:11:12 -> 14:11:** # # 14:11:12 -> 14:11:**
elif key in settings.OBFUSCATE_FIELDS_SEP: # elif key in settings.OBFUSCATE_FIELDS_SEP:
if "-" in value: # if "-" in value:
sep = "-" # sep = "-"
value_spl = value.split("-") # value_spl = value.split("-")
hide_num = settings.OBFUSCATE_DASH_NUM # hide_num = settings.OBFUSCATE_DASH_NUM
elif ":" in value: # elif ":" in value:
sep = ":" # sep = ":"
value_spl = value.split(":") # value_spl = value.split(":")
hide_num = settings.OBFUSCATE_COLON_NUM # hide_num = settings.OBFUSCATE_COLON_NUM
first_part = value_spl[:hide_num] # first_part = value_spl[:hide_num]
second_part = value_spl[hide_num:] # second_part = value_spl[hide_num:]
for index_x, x in enumerate(second_part): # for index_x, x in enumerate(second_part):
x_len = len(x) # x_len = len(x)
second_part[index_x] = "*" * x_len # second_part[index_x] = "*" * x_len
result = sep.join([*first_part, *second_part]) # result = sep.join([*first_part, *second_part])
data[index][key] = result # data[index][key] = result
for key in settings.COMBINE_FIELDS: # for key in settings.COMBINE_FIELDS:
for index, item in enumerate(data): # for index, item in enumerate(data):
if key in item: # if key in item:
k1, k2 = settings.COMBINE_FIELDS[key] # k1, k2 = settings.COMBINE_FIELDS[key]
if k1 in item and k2 in item: # if k1 in item and k2 in item:
data[index][key] = item[k1] + item[k2] # data[index][key] = item[k1] + item[k2]
def hash_list(user, data, hash_keys=False): # def hash_list(user, data, hash_keys=False):
""" # """
Hash a list of dicts or a list with SipHash42. # Hash a list of dicts or a list with SipHash42.
""" # """
if user.has_perm("core.bypass_hashing"): # if user.has_perm("core.bypass_hashing"):
return # return
cache = "cache.hash" # cache = "cache.hash"
hash_table = {} # hash_table = {}
if isinstance(data, dict): # if isinstance(data, dict):
data_copy = [{x: data[x]} for x in data] # data_copy = [{x: data[x]} for x in data]
else: # else:
data_copy = type(data)((data)) # data_copy = type(data)((data))
for index, item in enumerate(data_copy): # for index, item in enumerate(data_copy):
if "src" in item: # if "src" in item:
if item["src"] in settings.SAFE_SOURCES: # if item["src"] in settings.SAFE_SOURCES:
continue # continue
if isinstance(item, dict): # if isinstance(item, dict):
for key, value in list(item.items()): # for key, value in list(item.items()):
if ( # if (
key not in settings.WHITELIST_FIELDS # key not in settings.WHITELIST_FIELDS
and key not in settings.NO_OBFUSCATE_PARAMS # and key not in settings.NO_OBFUSCATE_PARAMS
): # ):
if isinstance(value, int): # if isinstance(value, int):
value = str(value) # value = str(value)
if isinstance(value, bool): # if isinstance(value, bool):
continue # continue
if value is None: # if value is None:
continue # continue
if hash_keys: # if hash_keys:
hashed = siphash(settings.HASHING_KEY, key) # hashed = siphash(settings.HASHING_KEY, key)
else: # else:
hashed = siphash(settings.HASHING_KEY, value) # hashed = siphash(settings.HASHING_KEY, value)
encoded = base36encode(hashed) # encoded = base36encode(hashed)
if encoded not in hash_table: # if encoded not in hash_table:
if hash_keys: # if hash_keys:
hash_table[encoded] = key # hash_table[encoded] = key
else: # else:
hash_table[encoded] = value # hash_table[encoded] = value
if hash_keys: # if hash_keys:
# Rename the dict key # # Rename the dict key
data[encoded] = data.pop(key) # data[encoded] = data.pop(key)
else: # else:
data[index][key] = encoded # data[index][key] = encoded
elif isinstance(item, str): # elif isinstance(item, str):
hashed = siphash(settings.HASHING_KEY, item) # hashed = siphash(settings.HASHING_KEY, item)
encoded = base36encode(hashed) # encoded = base36encode(hashed)
if encoded not in hash_table: # if encoded not in hash_table:
hash_table[encoded] = item # hash_table[encoded] = item
data[index] = encoded # data[index] = encoded
if hash_table: # if hash_table:
r.hmset(cache, hash_table) # r.hmset(cache, hash_table)
def hash_lookup(user, data_dict, supplementary_data=None): # def hash_lookup(user, data_dict, supplementary_data=None):
cache = "cache.hash" # cache = "cache.hash"
hash_list = SortedSet() # hash_list = SortedSet()
denied = [] # denied = []
for key, value in list(data_dict.items()): # for key, value in list(data_dict.items()):
if "source" in data_dict: # if "source" in data_dict:
if data_dict["source"] in settings.SAFE_SOURCES: # if data_dict["source"] in settings.SAFE_SOURCES:
continue # continue
if "src" in data_dict: # if "src" in data_dict:
if data_dict["src"] in settings.SAFE_SOURCES: # if data_dict["src"] in settings.SAFE_SOURCES:
continue # continue
if supplementary_data: # if supplementary_data:
if "source" in supplementary_data: # if "source" in supplementary_data:
if supplementary_data["source"] in settings.SAFE_SOURCES: # if supplementary_data["source"] in settings.SAFE_SOURCES:
continue # continue
if key in settings.SEARCH_FIELDS_DENY: # if key in settings.SEARCH_FIELDS_DENY:
if not user.has_perm("core.bypass_hashing"): # if not user.has_perm("core.bypass_hashing"):
data_dict[key] = SearchDenied(key=key, value=data_dict[key]) # data_dict[key] = SearchDenied(key=key, value=data_dict[key])
denied.append(data_dict[key]) # denied.append(data_dict[key])
if ( # if (
key not in settings.WHITELIST_FIELDS # key not in settings.WHITELIST_FIELDS
and key not in settings.NO_OBFUSCATE_PARAMS # and key not in settings.NO_OBFUSCATE_PARAMS
): # ):
if not value: # if not value:
continue # continue
# hashes = re.findall("\|([^\|]*)\|", value) # noqa # # hashes = re.findall("\|([^\|]*)\|", value) # noqa
if isinstance(value, str): # if isinstance(value, str):
hashes = re.findall("[A-Z0-9]{12,13}", value) # hashes = re.findall("[A-Z0-9]{12,13}", value)
elif isinstance(value, dict): # elif isinstance(value, dict):
hashes = [] # hashes = []
for key, value in value.items(): # for key, value in value.items():
if not value: # if not value:
continue # continue
hashes_iter = re.findall("[A-Z0-9]{12,13}", value) # hashes_iter = re.findall("[A-Z0-9]{12,13}", value)
for h in hashes_iter: # for h in hashes_iter:
hashes.append(h) # hashes.append(h)
if not hashes: # if not hashes:
# Otherwise the user could inject plaintext search queries # # Otherwise the user could inject plaintext search queries
if not user.has_perm("core.bypass_hashing"): # if not user.has_perm("core.bypass_hashing"):
data_dict[key] = SearchDenied(key=key, value=data_dict[key]) # data_dict[key] = SearchDenied(key=key, value=data_dict[key])
denied.append(data_dict[key]) # denied.append(data_dict[key])
continue # continue
else: # else:
# There are hashes here but there shouldn't be! # # There are hashes here but there shouldn't be!
if key in settings.TAG_SEARCH_DENY: # if key in settings.TAG_SEARCH_DENY:
data_dict[key] = LookupDenied(key=key, value=data_dict[key]) # data_dict[key] = LookupDenied(key=key, value=data_dict[key])
denied.append(data_dict[key]) # denied.append(data_dict[key])
continue # continue
for hash in hashes: # for hash in hashes:
hash_list.add(hash) # hash_list.add(hash)
if hash_list: # if hash_list:
values = r.hmget(cache, *hash_list) # values = r.hmget(cache, *hash_list)
if not values: # if not values:
return # return
for index, val in enumerate(values): # for index, val in enumerate(values):
if val is None: # if val is None:
values[index] = b"ERR" # values[index] = b"ERR"
values = [x.decode() for x in values] # values = [x.decode() for x in values]
total = dict(zip(hash_list, values)) # total = dict(zip(hash_list, values))
for key in data_dict.keys(): # for key in data_dict.keys():
for hash in total: # for hash in total:
if data_dict[key]: # if data_dict[key]:
if isinstance(data_dict[key], str): # if isinstance(data_dict[key], str):
if hash in data_dict[key]: # if hash in data_dict[key]:
data_dict[key] = data_dict[key].replace( # data_dict[key] = data_dict[key].replace(
f"{hash}", total[hash] # f"{hash}", total[hash]
) # )
elif isinstance(data_dict[key], dict): # elif isinstance(data_dict[key], dict):
for k2, v2 in data_dict[key].items(): # for k2, v2 in data_dict[key].items():
if hash in v2: # if hash in v2:
data_dict[key][k2] = v2.replace(f"{hash}", total[hash]) # data_dict[key][k2] = v2.repl
return denied # ace(f"{hash}", total[hash])
# return denied
def encrypt_list(user, data, secret): # def encrypt_list(user, data, secret):
if user.has_perm("core.bypass_encryption"): # if user.has_perm("core.bypass_encryption"):
return # return
cipher = Cipher(algorithms.AES(secret), ECB()) # cipher = Cipher(algorithms.AES(secret), ECB())
for index, item in enumerate(data): # for index, item in enumerate(data):
for key, value in item.items(): # for key, value in item.items():
if key not in settings.WHITELIST_FIELDS: # if key not in settings.WHITELIST_FIELDS:
encryptor = cipher.encryptor() # encryptor = cipher.encryptor()
if isinstance(value, int): # if isinstance(value, int):
value = str(value) # value = str(value)
if isinstance(value, bool): # if isinstance(value, bool):
continue # continue
if value is None: # if value is None:
continue # continue
decoded = value.encode("utf8", "replace") # decoded = value.encode("utf8", "replace")
length = 16 - (len(decoded) % 16) # length = 16 - (len(decoded) % 16)
decoded += bytes([length]) * length # decoded += bytes([length]) * length
ct = encryptor.update(decoded) + encryptor.finalize() # ct = encryptor.update(decoded) + encryptor.finalize()
final_str = b64encode(ct) # final_str = b64encode(ct)
data[index][key] = final_str.decode("utf-8", "replace") # data[index][key] = final_str.decode("utf-8", "replace")

100
core/views/ui/drilldown.py
View File

@ -1,7 +1,6 @@
import json import json
import urllib import urllib
import uuid import uuid
from copy import deepcopy
from django.conf import settings from django.conf import settings
from django.http import HttpResponse, JsonResponse from django.http import HttpResponse, JsonResponse
@ -20,9 +19,10 @@ from core.lib.threshold import (
get_chans, get_chans,
get_users, get_users,
) )
from core.views.helpers import hash_list, hash_lookup, randomise_list
from core.views.ui.tables import DrilldownTable from core.views.ui.tables import DrilldownTable
# from copy import deepcopy
def parse_dates(dates): def parse_dates(dates):
spl = dates.split(" - ") spl = dates.split(" - ")
@ -313,14 +313,14 @@ class DrilldownContextModal(APIView):
query_params[key] = None query_params[key] = None
# Lookup the hash values but don't disclose them to the user # Lookup the hash values but don't disclose them to the user
if settings.HASHING: # if settings.HASHING:
if query_params["source"] not in settings.SAFE_SOURCES: # if query_params["source"] not in settings.SAFE_SOURCES:
SAFE_PARAMS = deepcopy(query_params) # SAFE_PARAMS = deepcopy(query_params)
hash_lookup(request.user, SAFE_PARAMS) # hash_lookup(request.user, SAFE_PARAMS)
else: # else:
SAFE_PARAMS = deepcopy(query_params) # SAFE_PARAMS = deepcopy(query_params)
else: # else:
SAFE_PARAMS = query_params # SAFE_PARAMS = query_params
type = None type = None
if request.user.is_superuser: if request.user.is_superuser:
@ -328,12 +328,12 @@ class DrilldownContextModal(APIView):
type = query_params["type"] type = query_params["type"]
if type == "znc": if type == "znc":
query_params["channel"] = "*status" query_params["channel"] = "*status"
SAFE_PARAMS["channel"] = "*status" # SAFE_PARAMS["channel"] = "*status"
if type in ["query", "notice"]: if type in ["query", "notice"]:
nicks_sensitive = [ nicks_sensitive = [
SAFE_PARAMS["channel"], query_params["channel"],
SAFE_PARAMS["nick"], query_params["nick"],
] # UNSAFE ] # UNSAFE
# nicks = [query_params["channel"], query_params["nick"]] # nicks = [query_params["channel"], query_params["nick"]]
query = True query = True
@ -344,17 +344,17 @@ class DrilldownContextModal(APIView):
and not type == "query" and not type == "query"
): ):
query_params["index"] = "main" query_params["index"] = "main"
SAFE_PARAMS["index"] = "main" # SAFE_PARAMS["index"] = "main"
if query_params["type"] in ["znc", "auth"]: if query_params["type"] in ["znc", "auth"]:
query = True query = True
if not request.user.is_superuser: if not request.user.is_superuser:
query_params["index"] = "main" query_params["index"] = "main"
SAFE_PARAMS["index"] = "main" # SAFE_PARAMS["index"] = "main"
query_params["sorting"] = "desc" query_params["sorting"] = "desc"
SAFE_PARAMS["sorting"] = "desc" # SAFE_PARAMS["sorting"] = "desc"
annotate = False annotate = False
if query_params["source"] == "irc": if query_params["source"] == "irc":
@ -363,10 +363,10 @@ class DrilldownContextModal(APIView):
# Create the query with the context helper # Create the query with the context helper
search_query = construct_query( search_query = construct_query(
query_params["index"], query_params["index"],
SAFE_PARAMS["net"], query_params["net"],
SAFE_PARAMS["channel"], query_params["channel"],
query_params["source"], query_params["source"],
SAFE_PARAMS["num"], query_params["num"],
size, size,
type=type, type=type,
nicks=nicks_sensitive, nicks=nicks_sensitive,
@ -374,7 +374,7 @@ class DrilldownContextModal(APIView):
results = query_results( results = query_results(
request, request,
SAFE_PARAMS, query_params,
annotate=annotate, annotate=annotate,
custom_query=search_query, custom_query=search_query,
reverse=True, reverse=True,
@ -384,15 +384,15 @@ class DrilldownContextModal(APIView):
if "message" in results: if "message" in results:
return render(request, self.template_name, results) return render(request, self.template_name, results)
if settings.HASHING: # we probably want to see the tokens # if settings.HASHING: # we probably want to see the tokens
if query_params["source"] not in settings.SAFE_SOURCES: # if query_params["source"] not in settings.SAFE_SOURCES:
if not request.user.has_perm("core.bypass_hashing"): # if not request.user.has_perm("core.bypass_hashing"):
for index, item in enumerate(results["object_list"]): # for index, item in enumerate(results["object_list"]):
if "tokens" in item: # if "tokens" in item:
results["object_list"][index]["msg"] = results[ # results["object_list"][index]["msg"] = results[
"object_list" # "object_list"
][index].pop("tokens") # ][index].pop("tokens")
# item["msg"] = item.pop("tokens") # # item["msg"] = item.pop("tokens")
# Make the time nicer # Make the time nicer
# for index, item in enumerate(results["object_list"]): # for index, item in enumerate(results["object_list"]):
@ -446,37 +446,35 @@ class ThresholdInfoModal(APIView):
# SAFE BLOCK # # SAFE BLOCK #
# Lookup the hash values but don't disclose them to the user # Lookup the hash values but don't disclose them to the user
if settings.HASHING: # if settings.HASHING:
SAFE_PARAMS = request.data.dict() # SAFE_PARAMS = request.data.dict()
hash_lookup(request.user, SAFE_PARAMS) # hash_lookup(request.user, SAFE_PARAMS)
safe_net = SAFE_PARAMS["net"]
safe_nick = SAFE_PARAMS["nick"] channels = get_chans(net, [nick])
safe_channel = SAFE_PARAMS["channel"] users = get_users(net, [nick])
channels = get_chans(safe_net, [safe_nick]) num_users = annotate_num_users(net, channels)
users = get_users(safe_net, [safe_channel]) num_chans = annotate_num_chans(net, users)
num_users = annotate_num_users(safe_net, channels)
num_chans = annotate_num_chans(safe_net, users)
if channels: if channels:
inter_users = get_users(safe_net, channels) inter_users = get_users(net, channels)
else: else:
inter_users = [] inter_users = []
if users: if users:
inter_chans = get_chans(safe_net, users) inter_chans = get_chans(net, users)
else: else:
inter_chans = [] inter_chans = []
if settings.HASHING: # if settings.HASHING:
hash_list(request.user, inter_chans) # hash_list(request.user, inter_chans)
hash_list(request.user, inter_users) # hash_list(request.user, inter_users)
hash_list(request.user, num_chans, hash_keys=True) # hash_list(request.user, num_chans, hash_keys=True)
hash_list(request.user, num_users, hash_keys=True) # hash_list(request.user, num_users, hash_keys=True)
hash_list(request.user, channels) # hash_list(request.user, channels)
hash_list(request.user, users) # hash_list(request.user, users)
if settings.RANDOMISATION: # if settings.RANDOMISATION:
randomise_list(request.user, num_chans) # randomise_list(request.user, num_chans)
randomise_list(request.user, num_users) # randomise_list(request.user, num_users)
# SAFE BLOCK END # # SAFE BLOCK END #