Remove redaction stuff
This commit is contained in:
@@ -1,21 +1,12 @@
|
||||
from copy import deepcopy
|
||||
from datetime import datetime, timedelta
|
||||
# from copy import deepcopy
|
||||
# from datetime import datetime, timedelta
|
||||
|
||||
from django.conf import settings
|
||||
from opensearchpy import OpenSearch
|
||||
from opensearchpy.exceptions import NotFoundError, RequestError
|
||||
|
||||
from core.lib.threshold import annotate_num_chans, annotate_num_users, annotate_online
|
||||
from core.views.helpers import (
|
||||
LookupDenied,
|
||||
SearchDenied,
|
||||
dedup_list,
|
||||
encrypt_list,
|
||||
hash_list,
|
||||
hash_lookup,
|
||||
obfuscate_list,
|
||||
randomise_list,
|
||||
)
|
||||
from core.views.helpers import dedup_list
|
||||
|
||||
# from json import dumps
|
||||
# pp = lambda x: print(dumps(x, indent=2))
|
||||
@@ -216,9 +207,7 @@ def construct_query(query, size, use_query_string=True, tokens=False):
|
||||
return query_base
|
||||
|
||||
|
||||
def run_main_query(
|
||||
client, user, query, custom_query=False, index=None, size=None, filter=True
|
||||
):
|
||||
def run_main_query(client, user, query, custom_query=False, index=None, size=None):
|
||||
"""
|
||||
Low level helper to run an ES query.
|
||||
Accept a user to pass it to the filter, so we can
|
||||
@@ -240,8 +229,7 @@ def run_main_query(
|
||||
except NotFoundError as err:
|
||||
print("OpenSearch error", err)
|
||||
return err
|
||||
if filter:
|
||||
filter_blacklisted(user, response)
|
||||
filter_blacklisted(user, response)
|
||||
return response
|
||||
|
||||
|
||||
@@ -319,27 +307,27 @@ def query_results(
|
||||
query_created = False
|
||||
|
||||
# Lookup the hash values but don't disclose them to the user
|
||||
denied = []
|
||||
if lookup_hashes:
|
||||
if settings.HASHING:
|
||||
query_params = deepcopy(query_params)
|
||||
denied_q = hash_lookup(request.user, query_params)
|
||||
denied.extend(denied_q)
|
||||
if tags:
|
||||
denied_t = hash_lookup(request.user, tags, query_params)
|
||||
denied.extend(denied_t)
|
||||
# denied = []
|
||||
# if lookup_hashes:
|
||||
# if settings.HASHING:
|
||||
# query_params = deepcopy(query_params)
|
||||
# denied_q = hash_lookup(request.user, query_params)
|
||||
# denied.extend(denied_q)
|
||||
# if tags:
|
||||
# denied_t = hash_lookup(request.user, tags, query_params)
|
||||
# denied.extend(denied_t)
|
||||
|
||||
message = "Permission denied: "
|
||||
for x in denied:
|
||||
if isinstance(x, SearchDenied):
|
||||
message += f"Search({x.key}: {x.value}) "
|
||||
elif isinstance(x, LookupDenied):
|
||||
message += f"Lookup({x.key}: {x.value}) "
|
||||
if denied:
|
||||
# message = [f"{i}" for i in message]
|
||||
# message = "\n".join(message)
|
||||
message_class = "danger"
|
||||
return {"message": message, "class": message_class}
|
||||
# message = "Permission denied: "
|
||||
# for x in denied:
|
||||
# if isinstance(x, SearchDenied):
|
||||
# message += f"Search({x.key}: {x.value}) "
|
||||
# elif isinstance(x, LookupDenied):
|
||||
# message += f"Lookup({x.key}: {x.value}) "
|
||||
# if denied:
|
||||
# # message = [f"{i}" for i in message]
|
||||
# # message = "\n".join(message)
|
||||
# message_class = "danger"
|
||||
# return {"message": message, "class": message_class}
|
||||
|
||||
if request.user.is_anonymous:
|
||||
sizes = settings.OPENSEARCH_MAIN_SIZES_ANON
|
||||
@@ -357,14 +345,36 @@ def query_results(
|
||||
source = None
|
||||
if "source" in query_params:
|
||||
source = query_params["source"]
|
||||
if source not in settings.OPENSEARCH_MAIN_SOURCES:
|
||||
|
||||
if source in settings.OPENSEARCH_SOURCES_RESTRICTED:
|
||||
if not request.user.has_perm("restricted_sources"):
|
||||
message = "Access denied"
|
||||
message_class = "danger"
|
||||
return {"message": message, "class": message_class}
|
||||
elif source not in settings.OPENSEARCH_MAIN_SOURCES:
|
||||
message = "Invalid source"
|
||||
message_class = "danger"
|
||||
return {"message": message, "class": message_class}
|
||||
if source != "all":
|
||||
add_bool.append({"src": source})
|
||||
|
||||
date_query = False
|
||||
if source == "all":
|
||||
source = None # the next block will populate it
|
||||
|
||||
if source:
|
||||
sources = [source]
|
||||
else:
|
||||
sources = settings.OPENSEARCH_MAIN_SOURCES
|
||||
if request.user.has_perm("restricted_sources"):
|
||||
for source_iter in settings.OPENSEARCH_SOURCES_RESTRICTED:
|
||||
sources.append(source_iter)
|
||||
else:
|
||||
sources = settings.OPENSEARCH_MAIN_SOURCES
|
||||
|
||||
add_top_tmp = {"bool": {"should": []}}
|
||||
for source_iter in sources:
|
||||
add_top_tmp["bool"]["should"].append({"match_phrase": {"src": source_iter}})
|
||||
add_top.append(add_top_tmp)
|
||||
|
||||
# date_query = False
|
||||
if set({"from_date", "to_date", "from_time", "to_time"}).issubset(
|
||||
query_params.keys()
|
||||
):
|
||||
@@ -378,40 +388,40 @@ def query_results(
|
||||
}
|
||||
}
|
||||
}
|
||||
date_query = True
|
||||
add_top.append(range_query)
|
||||
|
||||
if date_query:
|
||||
if settings.DELAY_RESULTS:
|
||||
if source not in settings.SAFE_SOURCES:
|
||||
if request.user.has_perm("core.bypass_delay"):
|
||||
add_top.append(range_query)
|
||||
else:
|
||||
delay_as_ts = datetime.now() - timedelta(
|
||||
days=settings.DELAY_DURATION
|
||||
)
|
||||
lt_as_ts = datetime.strptime(
|
||||
range_query["range"]["ts"]["lt"], "%Y-%m-%dT%H:%MZ"
|
||||
)
|
||||
if lt_as_ts > delay_as_ts:
|
||||
range_query["range"]["ts"][
|
||||
"lt"
|
||||
] = f"now-{settings.DELAY_DURATION}d"
|
||||
add_top.append(range_query)
|
||||
else:
|
||||
add_top.append(range_query)
|
||||
else:
|
||||
if settings.DELAY_RESULTS:
|
||||
if source not in settings.SAFE_SOURCES:
|
||||
if not request.user.has_perm("core.bypass_delay"):
|
||||
range_query = {
|
||||
"range": {
|
||||
"ts": {
|
||||
# "gt": ,
|
||||
"lt": f"now-{settings.DELAY_DURATION}d",
|
||||
}
|
||||
}
|
||||
}
|
||||
add_top.append(range_query)
|
||||
# if date_query:
|
||||
# if settings.DELAY_RESULTS:
|
||||
# if source not in settings.SAFE_SOURCES:
|
||||
# if request.user.has_perm("core.bypass_delay"):
|
||||
# add_top.append(range_query)
|
||||
# else:
|
||||
# delay_as_ts = datetime.now() - timedelta(
|
||||
# days=settings.DELAY_DURATION
|
||||
# )
|
||||
# lt_as_ts = datetime.strptime(
|
||||
# range_query["range"]["ts"]["lt"], "%Y-%m-%dT%H:%MZ"
|
||||
# )
|
||||
# if lt_as_ts > delay_as_ts:
|
||||
# range_query["range"]["ts"][
|
||||
# "lt"
|
||||
# ] = f"now-{settings.DELAY_DURATION}d"
|
||||
# add_top.append(range_query)
|
||||
# else:
|
||||
# add_top.append(range_query)
|
||||
# else:
|
||||
# if settings.DELAY_RESULTS:
|
||||
# if source not in settings.SAFE_SOURCES:
|
||||
# if not request.user.has_perm("core.bypass_delay"):
|
||||
# range_query = {
|
||||
# "range": {
|
||||
# "ts": {
|
||||
# # "gt": ,
|
||||
# "lt": f"now-{settings.DELAY_DURATION}d",
|
||||
# }
|
||||
# }
|
||||
# }
|
||||
# add_top.append(range_query)
|
||||
|
||||
if "sorting" in query_params:
|
||||
sorting = query_params["sorting"]
|
||||
@@ -469,13 +479,13 @@ def query_results(
|
||||
query_created = True
|
||||
elif "query_full" in query_params:
|
||||
query_full = query_params["query_full"]
|
||||
if request.user.has_perm("core.query_search"):
|
||||
search_query = construct_query(query_full, size)
|
||||
query_created = True
|
||||
else:
|
||||
message = "You cannot search by query string"
|
||||
message_class = "danger"
|
||||
return {"message": message, "class": message_class}
|
||||
# if request.user.has_perm("core.query_search"):
|
||||
search_query = construct_query(query_full, size)
|
||||
query_created = True
|
||||
# else:
|
||||
# message = "You cannot search by query string"
|
||||
# message_class = "danger"
|
||||
# return {"message": message, "class": message_class}
|
||||
else:
|
||||
if custom_query:
|
||||
search_query = custom_query
|
||||
@@ -542,9 +552,6 @@ def query_results(
|
||||
else:
|
||||
index = settings.OPENSEARCH_INDEX_MAIN
|
||||
|
||||
filter = True
|
||||
if source in settings.SAFE_SOURCES:
|
||||
filter = False
|
||||
results = run_main_query(
|
||||
client,
|
||||
request.user, # passed through run_main_query to filter_blacklisted
|
||||
@@ -552,7 +559,6 @@ def query_results(
|
||||
custom_query=True,
|
||||
index=index,
|
||||
size=size,
|
||||
filter=filter,
|
||||
)
|
||||
if not results:
|
||||
return False
|
||||
@@ -585,18 +591,18 @@ def query_results(
|
||||
dedup_fields = ["msg", "nick", "ident", "host", "net", "channel"]
|
||||
results_parsed = dedup_list(results_parsed, dedup_fields)
|
||||
|
||||
if source not in settings.SAFE_SOURCES:
|
||||
if settings.ENCRYPTION:
|
||||
encrypt_list(request.user, results_parsed, settings.ENCRYPTION_KEY)
|
||||
# if source not in settings.SAFE_SOURCES:
|
||||
# if settings.ENCRYPTION:
|
||||
# encrypt_list(request.user, results_parsed, settings.ENCRYPTION_KEY)
|
||||
|
||||
if settings.HASHING:
|
||||
hash_list(request.user, results_parsed)
|
||||
# if settings.HASHING:
|
||||
# hash_list(request.user, results_parsed)
|
||||
|
||||
if settings.OBFUSCATION:
|
||||
obfuscate_list(request.user, results_parsed)
|
||||
# if settings.OBFUSCATION:
|
||||
# obfuscate_list(request.user, results_parsed)
|
||||
|
||||
if settings.RANDOMISATION:
|
||||
randomise_list(request.user, results_parsed)
|
||||
# if settings.RANDOMISATION:
|
||||
# randomise_list(request.user, results_parsed)
|
||||
|
||||
# process_list(results)
|
||||
|
||||
@@ -612,14 +618,14 @@ def query_results(
|
||||
context["exemption"] = results["exemption"]
|
||||
if query:
|
||||
context["query"] = query
|
||||
if settings.DELAY_RESULTS:
|
||||
if source not in settings.SAFE_SOURCES:
|
||||
if not request.user.has_perm("core.bypass_delay"):
|
||||
context["delay"] = settings.DELAY_DURATION
|
||||
if settings.RANDOMISATION:
|
||||
if source not in settings.SAFE_SOURCES:
|
||||
if not request.user.has_perm("core.bypass_randomisation"):
|
||||
context["randomised"] = True
|
||||
# if settings.DELAY_RESULTS:
|
||||
# if source not in settings.SAFE_SOURCES:
|
||||
# if not request.user.has_perm("core.bypass_delay"):
|
||||
# context["delay"] = settings.DELAY_DURATION
|
||||
# if settings.RANDOMISATION:
|
||||
# if source not in settings.SAFE_SOURCES:
|
||||
# if not request.user.has_perm("core.bypass_randomisation"):
|
||||
# context["randomised"] = True
|
||||
return context
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user