Improve data security by mandating token search
This commit is contained in:
parent
e85fa910aa
commit
3f02c61463
|
@ -16,6 +16,45 @@ OPENSEARCH_MAIN_WILDCARD_ANON = False
|
||||||
OPENSEARCH_MAIN_SOURCES = ["irc", "dis", "all"]
|
OPENSEARCH_MAIN_SOURCES = ["irc", "dis", "all"]
|
||||||
DRILLDOWN_RESULTS_PER_PAGE = 15
|
DRILLDOWN_RESULTS_PER_PAGE = 15
|
||||||
|
|
||||||
|
# Encryption
|
||||||
|
ENCRYPTION = False
|
||||||
|
ENCRYPTION_KEY = b""
|
||||||
|
|
||||||
|
# Hashing
|
||||||
|
HASHING = True
|
||||||
|
HASHING_KEY = "xxx"
|
||||||
|
|
||||||
|
# Common to encryption and hashing
|
||||||
|
WHITELIST_FIELDS = [
|
||||||
|
"ts",
|
||||||
|
"date",
|
||||||
|
"time",
|
||||||
|
"sentiment",
|
||||||
|
"version_sentiment",
|
||||||
|
"tokens",
|
||||||
|
"num_chans",
|
||||||
|
"num_users",
|
||||||
|
"tokens",
|
||||||
|
"src",
|
||||||
|
"exemption",
|
||||||
|
"hidden",
|
||||||
|
]
|
||||||
|
|
||||||
|
# Don't obfuscate these parameters, or lookup hashes in them
|
||||||
|
NO_OBFUSCATE_PARAMS = [
|
||||||
|
"query",
|
||||||
|
"query_full",
|
||||||
|
"size",
|
||||||
|
"source",
|
||||||
|
"sorting",
|
||||||
|
"tags",
|
||||||
|
"index",
|
||||||
|
"dedup",
|
||||||
|
"check_sentiment",
|
||||||
|
"sentiment_method",
|
||||||
|
"dates",
|
||||||
|
]
|
||||||
|
|
||||||
OPENSEARCH_BLACKLISTED = {}
|
OPENSEARCH_BLACKLISTED = {}
|
||||||
|
|
||||||
# URLs
|
# URLs
|
||||||
|
|
|
@ -5,7 +5,16 @@ from opensearchpy import OpenSearch
|
||||||
from opensearchpy.exceptions import NotFoundError, RequestError
|
from opensearchpy.exceptions import NotFoundError, RequestError
|
||||||
|
|
||||||
from core.lib.threshold import annotate_num_chans, annotate_num_users, annotate_online
|
from core.lib.threshold import annotate_num_chans, annotate_num_users, annotate_online
|
||||||
from core.views.helpers import dedup_list, encrypt_list, hash_list, hash_lookup
|
from core.views.helpers import (
|
||||||
|
SearchDenied,
|
||||||
|
dedup_list,
|
||||||
|
encrypt_list,
|
||||||
|
hash_list,
|
||||||
|
hash_lookup,
|
||||||
|
)
|
||||||
|
|
||||||
|
# from json import dumps
|
||||||
|
# pp = lambda x: print(dumps(x, indent=2))
|
||||||
|
|
||||||
|
|
||||||
def initialise_opensearch():
|
def initialise_opensearch():
|
||||||
|
@ -141,18 +150,17 @@ def filter_blacklisted(user, response):
|
||||||
response["hits"]["hits"] = [hit for hit in response["hits"]["hits"] if hit]
|
response["hits"]["hits"] = [hit for hit in response["hits"]["hits"] if hit]
|
||||||
|
|
||||||
|
|
||||||
def construct_query(query, size):
|
def construct_query(query, size, use_query_string=True, tokens=False):
|
||||||
"""
|
"""
|
||||||
Accept some query parameters and construct an OpenSearch query.
|
Accept some query parameters and construct an OpenSearch query.
|
||||||
"""
|
"""
|
||||||
if not size:
|
if not size:
|
||||||
size = 5
|
size = 5
|
||||||
query = {
|
query_base = {
|
||||||
"size": size,
|
"size": size,
|
||||||
"query": {
|
"query": {"bool": {"must": []}},
|
||||||
"bool": {
|
}
|
||||||
"must": [
|
query_string = {
|
||||||
{
|
|
||||||
"query_string": {
|
"query_string": {
|
||||||
"query": query,
|
"query": query,
|
||||||
# "fields": fields,
|
# "fields": fields,
|
||||||
|
@ -177,11 +185,31 @@ def construct_query(query, size):
|
||||||
"auto_generate_synonyms_phrase_query": True,
|
"auto_generate_synonyms_phrase_query": True,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
]
|
query_tokens = {
|
||||||
|
"simple_query_string": {
|
||||||
|
# "tokens": query,
|
||||||
|
"query": query,
|
||||||
|
"fields": ["tokens"],
|
||||||
|
"flags": "ALL",
|
||||||
|
"fuzzy_transpositions": True,
|
||||||
|
"fuzzy_max_expansions": 50,
|
||||||
|
"fuzzy_prefix_length": 0,
|
||||||
|
"default_operator": "and",
|
||||||
|
"analyzer": "standard",
|
||||||
|
"lenient": True,
|
||||||
|
"boost": 1,
|
||||||
|
"quote_field_suffix": "",
|
||||||
|
"analyze_wildcard": False,
|
||||||
|
"auto_generate_synonyms_phrase_query": False,
|
||||||
}
|
}
|
||||||
},
|
|
||||||
}
|
}
|
||||||
return query
|
if tokens:
|
||||||
|
query_base["query"]["bool"]["must"].append(query_tokens)
|
||||||
|
# query["query"]["bool"]["must"].append(query_string)
|
||||||
|
# query["query"]["bool"]["must"][0]["query_string"]["fields"] = ["tokens"]
|
||||||
|
elif use_query_string:
|
||||||
|
query_base["query"]["bool"]["must"].append(query_string)
|
||||||
|
return query_base
|
||||||
|
|
||||||
|
|
||||||
def run_main_query(client, user, query, custom_query=False, index=None, size=None):
|
def run_main_query(client, user, query, custom_query=False, index=None, size=None):
|
||||||
|
@ -261,6 +289,7 @@ def query_results(
|
||||||
dedup=False,
|
dedup=False,
|
||||||
dedup_fields=None,
|
dedup_fields=None,
|
||||||
lookup_hashes=True,
|
lookup_hashes=True,
|
||||||
|
tags=None,
|
||||||
):
|
):
|
||||||
"""
|
"""
|
||||||
API helper to alter the OpenSearch return format into something
|
API helper to alter the OpenSearch return format into something
|
||||||
|
@ -276,12 +305,15 @@ def query_results(
|
||||||
add_top = []
|
add_top = []
|
||||||
add_top_negative = []
|
add_top_negative = []
|
||||||
sort = None
|
sort = None
|
||||||
|
query_created = False
|
||||||
|
|
||||||
# Lookup the hash values but don't disclose them to the user
|
# Lookup the hash values but don't disclose them to the user
|
||||||
if lookup_hashes:
|
if lookup_hashes:
|
||||||
if settings.HASHING:
|
if settings.HASHING:
|
||||||
query_params = deepcopy(query_params)
|
query_params = deepcopy(query_params)
|
||||||
hash_lookup(query_params)
|
hash_lookup(request.user, query_params)
|
||||||
|
if tags:
|
||||||
|
hash_lookup(request.user, tags)
|
||||||
|
|
||||||
if request.user.is_anonymous:
|
if request.user.is_anonymous:
|
||||||
sizes = settings.OPENSEARCH_MAIN_SIZES_ANON
|
sizes = settings.OPENSEARCH_MAIN_SIZES_ANON
|
||||||
|
@ -366,15 +398,53 @@ def query_results(
|
||||||
range_query_precise["match"]["sentiment"] = 0
|
range_query_precise["match"]["sentiment"] = 0
|
||||||
add_top_negative.append(range_query_precise)
|
add_top_negative.append(range_query_precise)
|
||||||
|
|
||||||
|
# Only one of query or query_full can be active at once
|
||||||
|
# We prefer query because it's simpler
|
||||||
if "query" in query_params:
|
if "query" in query_params:
|
||||||
query = query_params["query"]
|
query = query_params["query"]
|
||||||
search_query = construct_query(query, size)
|
search_query = construct_query(query, size, tokens=True)
|
||||||
|
query_created = True
|
||||||
|
elif "query_full" in query_params:
|
||||||
|
query_full = query_params["query_full"]
|
||||||
|
if request.user.has_perm("query_search"):
|
||||||
|
search_query = construct_query(query_full, size)
|
||||||
|
query_created = True
|
||||||
|
else:
|
||||||
|
message = "You cannot search by query string"
|
||||||
|
message_class = "danger"
|
||||||
|
return {"message": message, "class": message_class}
|
||||||
else:
|
else:
|
||||||
if custom_query:
|
if custom_query:
|
||||||
search_query = custom_query
|
search_query = custom_query
|
||||||
|
|
||||||
|
if tags:
|
||||||
|
# Get a blank search query
|
||||||
|
if not query_created:
|
||||||
|
search_query = construct_query(None, size, use_query_string=False)
|
||||||
|
query_created = True
|
||||||
|
for tagname, tagvalue in tags.items():
|
||||||
|
add_bool.append({tagname: tagvalue})
|
||||||
|
|
||||||
|
required_any = ["query_full", "query", "tags"]
|
||||||
|
if not any([field in query_params.keys() for field in required_any]):
|
||||||
|
if not custom_query:
|
||||||
|
message = "Empty query!"
|
||||||
|
message_class = "warning"
|
||||||
|
return {"message": message, "class": message_class}
|
||||||
|
|
||||||
if add_bool:
|
if add_bool:
|
||||||
|
# if "bool" not in search_query["query"]:
|
||||||
|
# search_query["query"]["bool"] = {}
|
||||||
|
# if "must" not in search_query["query"]["bool"]:
|
||||||
|
# search_query["query"]["bool"] = {"must": []}
|
||||||
|
|
||||||
for item in add_bool:
|
for item in add_bool:
|
||||||
search_query["query"]["bool"]["must"].append({"match": item})
|
k, v = list(item.items())[0]
|
||||||
|
if isinstance(v, SearchDenied):
|
||||||
|
message = f"Access denied: search by protected field {k}: {v.value}"
|
||||||
|
message_class = "danger"
|
||||||
|
return {"message": message, "class": message_class}
|
||||||
|
search_query["query"]["bool"]["must"].append({"match_phrase": item})
|
||||||
if add_top:
|
if add_top:
|
||||||
for item in add_top:
|
for item in add_top:
|
||||||
search_query["query"]["bool"]["must"].append(item)
|
search_query["query"]["bool"]["must"].append(item)
|
||||||
|
@ -398,7 +468,6 @@ def query_results(
|
||||||
return {
|
return {
|
||||||
"message": message,
|
"message": message,
|
||||||
"class": message_class,
|
"class": message_class,
|
||||||
"params": query_params,
|
|
||||||
}
|
}
|
||||||
if index == "meta":
|
if index == "meta":
|
||||||
index = settings.OPENSEARCH_INDEX_META
|
index = settings.OPENSEARCH_INDEX_META
|
||||||
|
@ -410,7 +479,6 @@ def query_results(
|
||||||
return {
|
return {
|
||||||
"message": message,
|
"message": message,
|
||||||
"class": message_class,
|
"class": message_class,
|
||||||
"params": query_params,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
else:
|
else:
|
||||||
|
@ -461,7 +529,6 @@ def query_results(
|
||||||
if not request.user.has_perm("view_plain"):
|
if not request.user.has_perm("view_plain"):
|
||||||
if settings.HASHING:
|
if settings.HASHING:
|
||||||
hash_list(request.user, results_parsed)
|
hash_list(request.user, results_parsed)
|
||||||
|
|
||||||
# process_list(reqults)
|
# process_list(reqults)
|
||||||
|
|
||||||
# IMPORTANT! - DO NOT PASS query_params to the user!
|
# IMPORTANT! - DO NOT PASS query_params to the user!
|
||||||
|
|
File diff suppressed because one or more lines are too long
|
@ -15,6 +15,7 @@
|
||||||
<link rel="stylesheet" href="{% static 'css/bulma-slider.min.css' %}">
|
<link rel="stylesheet" href="{% static 'css/bulma-slider.min.css' %}">
|
||||||
<link rel="stylesheet" href="{% static 'css/bulma-calendar.min.css' %}">
|
<link rel="stylesheet" href="{% static 'css/bulma-calendar.min.css' %}">
|
||||||
<link rel="stylesheet" href="{% static 'css/bulma-tagsinput.min.css' %}">
|
<link rel="stylesheet" href="{% static 'css/bulma-tagsinput.min.css' %}">
|
||||||
|
<link rel="stylesheet" href="{% static 'css/bulma-switch.min.css' %}">
|
||||||
<script src="{% static 'js/bulma-calendar.min.js' %}" integrity="sha384-DThNif0xGXbopX7+PE+UabkuClfI/zELNhaVqoGLutaWB76dyMw0vIQBGmUxSfVQ" crossorigin="anonymous"></script>
|
<script src="{% static 'js/bulma-calendar.min.js' %}" integrity="sha384-DThNif0xGXbopX7+PE+UabkuClfI/zELNhaVqoGLutaWB76dyMw0vIQBGmUxSfVQ" crossorigin="anonymous"></script>
|
||||||
<script src="{% static 'js/bulma-slider.min.js' %}" integrity="sha384-wbyps8iLG8QzJE02viYc/27BtT5HSa11+b5V7QPR1/huVuA8f4LRTNGc82qAIeIZ" crossorigin="anonymous"></script>
|
<script src="{% static 'js/bulma-slider.min.js' %}" integrity="sha384-wbyps8iLG8QzJE02viYc/27BtT5HSa11+b5V7QPR1/huVuA8f4LRTNGc82qAIeIZ" crossorigin="anonymous"></script>
|
||||||
<script defer src="{% static 'js/htmx.min.js' %}" integrity="sha384-cZuAZ+ZbwkNRnrKi05G/fjBX+azI9DNOkNYysZ0I/X5ZFgsmMiBXgDZof30F5ofc" crossorigin="anonymous"></script>
|
<script defer src="{% static 'js/htmx.min.js' %}" integrity="sha384-cZuAZ+ZbwkNRnrKi05G/fjBX+azI9DNOkNYysZ0I/X5ZFgsmMiBXgDZof30F5ofc" crossorigin="anonymous"></script>
|
||||||
|
|
|
@ -39,60 +39,23 @@
|
||||||
} catch {
|
} catch {
|
||||||
var value = spl[1];
|
var value = spl[1];
|
||||||
}
|
}
|
||||||
populateSearch(field, value);
|
|
||||||
return `${field}: ${value}`;
|
return `${field}: ${value}`;
|
||||||
});
|
});
|
||||||
inputTags.BulmaTagsInput().on('after.remove', function(item) {
|
inputTags.BulmaTagsInput().on('after.remove', function(item) {
|
||||||
var spl = item.split(": ");
|
var spl = item.split(": ");
|
||||||
var field = spl[0];
|
var field = spl[0];
|
||||||
var value = spl[1].trim();
|
var value = spl[1].trim();
|
||||||
populateSearch(field, value);
|
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
function populateSearch(field, value) {
|
function populateSearch(field, value) {
|
||||||
var queryElement = document.getElementById('query');
|
var inputTags = document.getElementById('tags');
|
||||||
|
inputTags.BulmaTagsInput().add(field+": "+value);
|
||||||
var present = true;
|
|
||||||
if (present == true) {
|
|
||||||
var combinations = [`${field}: "${value}"`,
|
|
||||||
`${field}: "${value}"`,
|
|
||||||
`${field}: ${value}`,
|
|
||||||
`${field}:${value}`,
|
|
||||||
`${field}:"${value}"`];
|
|
||||||
var toAppend = ` AND ${field}: "${value}"`;
|
|
||||||
} else {
|
|
||||||
var combinations = [`NOT ${field}: "${value}"`,
|
|
||||||
`NOT ${field}: "${value}"`,
|
|
||||||
`NOT ${field}: ${value}`,
|
|
||||||
`NOT ${field}:${value}`,
|
|
||||||
`NOT ${field}:"${value}"`];
|
|
||||||
}
|
|
||||||
var contains = combinations.some(elem => queryElement.value.includes(elem));
|
|
||||||
if (!contains) {
|
|
||||||
queryElement.value+=toAppend;
|
|
||||||
} else {
|
|
||||||
for (var index in combinations) {
|
|
||||||
combination = combinations[index];
|
|
||||||
queryElement.value = queryElement.value.replaceAll("AND "+combination, "");
|
|
||||||
queryElement.value = queryElement.value.replaceAll(combination, "");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (field == "src") {
|
|
||||||
document.getElementById("source").selectedIndex = 2;
|
|
||||||
}
|
|
||||||
if (queryElement.value.startsWith(" AND ")) {
|
|
||||||
queryElement.value = queryElement.value.replace(" AND ", "");
|
|
||||||
}
|
|
||||||
if (queryElement.value.startsWith("AND ")) {
|
|
||||||
queryElement.value = queryElement.value.replace("AND ", "");
|
|
||||||
}
|
|
||||||
htmx.trigger("#search", "click");
|
htmx.trigger("#search", "click");
|
||||||
}
|
}
|
||||||
</script>
|
</script>
|
||||||
<div>
|
<div>
|
||||||
{% include 'partials/notify.html' %}
|
{% include 'partials/notify.html' %}
|
||||||
<form method="POST" hx-post="{% url 'search' %}"
|
<form class="skipEmptyFields" method="POST" hx-post="{% url 'search' %}"
|
||||||
hx-trigger="change"
|
hx-trigger="change"
|
||||||
hx-target="#results"
|
hx-target="#results"
|
||||||
hx-swap="innerHTML"
|
hx-swap="innerHTML"
|
||||||
|
@ -102,12 +65,17 @@
|
||||||
<div class="columns">
|
<div class="columns">
|
||||||
<div class="column">
|
<div class="column">
|
||||||
<div class="field has-addons">
|
<div class="field has-addons">
|
||||||
<div class="control is-expanded has-icons-left">
|
<div id="query" class="control is-expanded has-icons-left">
|
||||||
<input
|
<input
|
||||||
hx-post="{% url 'search' %}"
|
hx-post="{% url 'search' %}"
|
||||||
hx-trigger="keyup changed delay:200ms"
|
hx-trigger="keyup changed delay:200ms"
|
||||||
hx-target="#results"
|
hx-target="#results"
|
||||||
hx-swap="innerHTML" id="query" name="query" value="{{ params.query }}" class="input" type="text" placeholder="msg: science AND nick: BillNye AND channel: #science">
|
hx-swap="innerHTML"
|
||||||
|
name="query"
|
||||||
|
value="{{ params.query }}"
|
||||||
|
class="input"
|
||||||
|
type="text"
|
||||||
|
placeholder="(science | tech | art) + (interest) -hello">
|
||||||
<span class="icon is-small is-left">
|
<span class="icon is-small is-left">
|
||||||
<i class="fas fa-magnifying-glass"></i>
|
<i class="fas fa-magnifying-glass"></i>
|
||||||
</span>
|
</span>
|
||||||
|
@ -166,6 +134,19 @@
|
||||||
</a>
|
</a>
|
||||||
</p>
|
</p>
|
||||||
</div>
|
</div>
|
||||||
|
<div class="control">
|
||||||
|
<div class="field">
|
||||||
|
<input
|
||||||
|
id="full_query"
|
||||||
|
type="checkbox"
|
||||||
|
class="switch"
|
||||||
|
{% if params.query_full is not None %}checked="checked"{% else %}none{% endif %}
|
||||||
|
data-script="on click toggle .is-hidden on #query_full">
|
||||||
|
<label
|
||||||
|
class="{% if not perms.core.query_search %}is-disabled{% endif %}"
|
||||||
|
for="full_query">Full query </label>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
<div class="column is-narrow">
|
<div class="column is-narrow">
|
||||||
<div class="field has-addons block">
|
<div class="field has-addons block">
|
||||||
|
@ -411,24 +392,45 @@
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
<div id="query_full" class="block {% if params.query_full is None %}is-hidden{% endif %}">
|
||||||
|
<div class="control is-expanded has-icons-left">
|
||||||
|
<input
|
||||||
|
hx-post="{% url 'search' %}"
|
||||||
|
hx-trigger="keyup changed delay:200ms"
|
||||||
|
hx-target="#results"
|
||||||
|
hx-swap="innerHTML"
|
||||||
|
name="query_full"
|
||||||
|
value="{{ params.query_full }}"
|
||||||
|
class="input"
|
||||||
|
type="text"
|
||||||
|
placeholder="msg: science AND nick: BillNye AND channel: #science">
|
||||||
|
<span class="icon is-small is-left">
|
||||||
|
<i class="fas fa-magnifying-glass"></i>
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="block">
|
||||||
|
<input
|
||||||
|
hx-trigger="change"
|
||||||
|
hx-post="{% url 'search' %}"
|
||||||
|
hx-target="#results"
|
||||||
|
hx-swap="innerHTML"
|
||||||
|
id="tags"
|
||||||
|
class="input"
|
||||||
|
type="tags"
|
||||||
|
name="tags"
|
||||||
|
placeholder="Add tags"
|
||||||
|
value="{{ params.tags }}">
|
||||||
|
</div>
|
||||||
<div class="is-hidden"></div>
|
<div class="is-hidden"></div>
|
||||||
</form>
|
</form>
|
||||||
</div>
|
</div>
|
||||||
<div class="block">
|
|
||||||
<input id="tags" class="input" type="tags" placeholder="Add query" value="{{ tags|joinsep:',' }}">
|
|
||||||
</div>
|
|
||||||
<div class="block">
|
<div class="block">
|
||||||
<div id="results">
|
<div id="results">
|
||||||
<!-- {% if results %}
|
|
||||||
{% include 'ui/drilldown/results.html' %}
|
|
||||||
{% endif %} -->
|
|
||||||
{% if table %}
|
|
||||||
{% include 'ui/drilldown/table_results.html' %}
|
{% include 'ui/drilldown/table_results.html' %}
|
||||||
{% else %}
|
|
||||||
<script>
|
<script>
|
||||||
setupTags();
|
setupTags();
|
||||||
</script>
|
</script>
|
||||||
{% endif %}
|
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
<div id="modals-here">
|
<div id="modals-here">
|
||||||
|
|
|
@ -37,23 +37,3 @@
|
||||||
{% endif %}
|
{% endif %}
|
||||||
{% include 'ui/drilldown/table_results_partial.html' %}
|
{% include 'ui/drilldown/table_results_partial.html' %}
|
||||||
{% endif %}
|
{% endif %}
|
||||||
{# Update the tags in case the user changed the query #}
|
|
||||||
{# Check for focus and refocus #}
|
|
||||||
<script>
|
|
||||||
var inputTags = document.getElementsByClassName('tags-input');
|
|
||||||
var inputBox = document.querySelector("[placeholder='Add query']");
|
|
||||||
var isFocused = (document.activeElement === inputBox);
|
|
||||||
for (index = 0; index < inputTags.length; index++) {
|
|
||||||
if (index == 0) {
|
|
||||||
inputTags[0].outerHTML = '<input id="tags" class="input" type="tags" placeholder="Add query" value="{{ tags|joinsep:',' }}">';
|
|
||||||
} else {
|
|
||||||
inputTags[index].remove();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// inputTags[0].outerHTML = '<input id="tags" class="input" type="tags" placeholder="Add query" value="{{ tags|joinsep:',' }}">';
|
|
||||||
setupTags();
|
|
||||||
var inputBox = document.querySelector("[placeholder='Add query']");
|
|
||||||
if (isFocused) {
|
|
||||||
inputBox.focus();
|
|
||||||
}
|
|
||||||
</script>
|
|
|
@ -143,7 +143,7 @@
|
||||||
</span>
|
</span>
|
||||||
</td>
|
</td>
|
||||||
{% elif column.name == 'tokens' %}
|
{% elif column.name == 'tokens' %}
|
||||||
<td class="{{ column.name }}">
|
<td class="{{ column.name }} wrap" style="max-width: 10em">
|
||||||
{{ cell|joinsep:',' }}
|
{{ cell|joinsep:',' }}
|
||||||
</td>
|
</td>
|
||||||
{% elif column.name == 'src' %}
|
{% elif column.name == 'src' %}
|
||||||
|
|
|
@ -10,6 +10,11 @@ from sortedcontainers import SortedSet
|
||||||
from core import r
|
from core import r
|
||||||
|
|
||||||
|
|
||||||
|
class SearchDenied:
|
||||||
|
def __init__(self, value):
|
||||||
|
self.value = value
|
||||||
|
|
||||||
|
|
||||||
def dedup_list(data, check_keys):
|
def dedup_list(data, check_keys):
|
||||||
"""
|
"""
|
||||||
Remove duplicate dictionaries from list.
|
Remove duplicate dictionaries from list.
|
||||||
|
@ -90,7 +95,10 @@ def hash_list(user, data, hash_keys=False):
|
||||||
for index, item in enumerate(data_copy):
|
for index, item in enumerate(data_copy):
|
||||||
if isinstance(item, dict):
|
if isinstance(item, dict):
|
||||||
for key, value in list(item.items()):
|
for key, value in list(item.items()):
|
||||||
if key not in settings.WHITELIST_FIELDS:
|
if (
|
||||||
|
key not in settings.WHITELIST_FIELDS
|
||||||
|
and key not in settings.NO_OBFUSCATE_PARAMS
|
||||||
|
):
|
||||||
if isinstance(value, int):
|
if isinstance(value, int):
|
||||||
value = str(value)
|
value = str(value)
|
||||||
if isinstance(value, bool):
|
if isinstance(value, bool):
|
||||||
|
@ -122,16 +130,33 @@ def hash_list(user, data, hash_keys=False):
|
||||||
r.hmset(cache, hash_table)
|
r.hmset(cache, hash_table)
|
||||||
|
|
||||||
|
|
||||||
def hash_lookup(data_dict):
|
def hash_lookup(user, data_dict):
|
||||||
cache = "cache.hash"
|
cache = "cache.hash"
|
||||||
hash_list = SortedSet()
|
hash_list = SortedSet()
|
||||||
for key, value in data_dict.items():
|
for key, value in list(data_dict.items()):
|
||||||
|
if (
|
||||||
|
key not in settings.WHITELIST_FIELDS
|
||||||
|
and key not in settings.NO_OBFUSCATE_PARAMS
|
||||||
|
):
|
||||||
if not value:
|
if not value:
|
||||||
continue
|
continue
|
||||||
# hashes = re.findall("\|([^\|]*)\|", value) # noqa
|
# hashes = re.findall("\|([^\|]*)\|", value) # noqa
|
||||||
|
if isinstance(value, str):
|
||||||
hashes = re.findall("[A-Z0-9]{12,13}", value)
|
hashes = re.findall("[A-Z0-9]{12,13}", value)
|
||||||
if not hashes:
|
elif isinstance(value, dict):
|
||||||
|
hashes = []
|
||||||
|
for key, value in value.items():
|
||||||
|
if not value:
|
||||||
continue
|
continue
|
||||||
|
hashes_iter = re.findall("[A-Z0-9]{12,13}", value)
|
||||||
|
for h in hashes_iter:
|
||||||
|
hashes.append(h)
|
||||||
|
if not hashes:
|
||||||
|
# Otherwise the user could inject plaintext search queries
|
||||||
|
if not user.has_perm("bypass_hashing"):
|
||||||
|
data_dict[key] = SearchDenied(value=data_dict[key])
|
||||||
|
# del data_dict[key]
|
||||||
|
|
||||||
for hash in hashes:
|
for hash in hashes:
|
||||||
hash_list.add(hash)
|
hash_list.add(hash)
|
||||||
|
|
||||||
|
@ -147,8 +172,17 @@ def hash_lookup(data_dict):
|
||||||
for key in data_dict.keys():
|
for key in data_dict.keys():
|
||||||
for hash in total:
|
for hash in total:
|
||||||
if data_dict[key]:
|
if data_dict[key]:
|
||||||
|
if isinstance(data_dict[key], str):
|
||||||
if hash in data_dict[key]:
|
if hash in data_dict[key]:
|
||||||
data_dict[key] = data_dict[key].replace(f"{hash}", total[hash])
|
print("Replacing", data_dict[key], "with", total[hash])
|
||||||
|
data_dict[key] = data_dict[key].replace(
|
||||||
|
f"{hash}", total[hash]
|
||||||
|
)
|
||||||
|
elif isinstance(data_dict[key], dict):
|
||||||
|
for k2, v2 in data_dict[key].items():
|
||||||
|
if hash in v2:
|
||||||
|
print("Replacing", v2, "with", total[hash])
|
||||||
|
data_dict[key][k2] = v2.replace(f"{hash}", total[hash])
|
||||||
|
|
||||||
|
|
||||||
def encrypt_list(user, data, secret):
|
def encrypt_list(user, data, secret):
|
||||||
|
|
|
@ -55,6 +55,21 @@ def create_tags(query):
|
||||||
return tags
|
return tags
|
||||||
|
|
||||||
|
|
||||||
|
def parse_tags(tags_pre):
|
||||||
|
"""
|
||||||
|
Parse the tags from the variable tags_pre.
|
||||||
|
"""
|
||||||
|
tags = {}
|
||||||
|
tags_spl = tags_pre.split(",")
|
||||||
|
if tags_spl:
|
||||||
|
for tag in tags_spl:
|
||||||
|
tag = tag.split(": ")
|
||||||
|
if len(tag) == 2:
|
||||||
|
key, val = tag
|
||||||
|
tags[key] = val
|
||||||
|
return tags
|
||||||
|
|
||||||
|
|
||||||
def make_table(context):
|
def make_table(context):
|
||||||
table = DrilldownTable(context["object_list"])
|
table = DrilldownTable(context["object_list"])
|
||||||
context["table"] = table
|
context["table"] = table
|
||||||
|
@ -78,6 +93,8 @@ def make_graph(results):
|
||||||
|
|
||||||
|
|
||||||
def drilldown_search(request, return_context=False, template=None):
|
def drilldown_search(request, return_context=False, template=None):
|
||||||
|
extra_params = {}
|
||||||
|
|
||||||
if not template:
|
if not template:
|
||||||
template_name = "ui/drilldown/table_results.html"
|
template_name = "ui/drilldown/table_results.html"
|
||||||
else:
|
else:
|
||||||
|
@ -105,6 +122,10 @@ def drilldown_search(request, return_context=False, template=None):
|
||||||
query_params.update(tmp_post)
|
query_params.update(tmp_post)
|
||||||
query_params.update(tmp_get)
|
query_params.update(tmp_get)
|
||||||
|
|
||||||
|
# URI we're passing to the template for linking
|
||||||
|
if "csrfmiddlewaretoken" in query_params:
|
||||||
|
del query_params["csrfmiddlewaretoken"]
|
||||||
|
|
||||||
# Parse the dates
|
# Parse the dates
|
||||||
if "dates" in query_params:
|
if "dates" in query_params:
|
||||||
dates = parse_dates(query_params["dates"])
|
dates = parse_dates(query_params["dates"])
|
||||||
|
@ -118,24 +139,34 @@ def drilldown_search(request, return_context=False, template=None):
|
||||||
query_params["to_time"] = dates["to_time"]
|
query_params["to_time"] = dates["to_time"]
|
||||||
|
|
||||||
if "query" in query_params:
|
if "query" in query_params:
|
||||||
context = query_results(request, query_params)
|
# Remove null values
|
||||||
|
if query_params["query"] == "":
|
||||||
|
del query_params["query"]
|
||||||
# Turn the query into tags for populating the taglist
|
# Turn the query into tags for populating the taglist
|
||||||
tags = create_tags(query_params["query"])
|
# tags = create_tags(query_params["query"])
|
||||||
context["tags"] = tags
|
# context["tags"] = tags
|
||||||
|
# else:
|
||||||
|
# context = {"object_list": []}
|
||||||
|
|
||||||
|
# Remove null values
|
||||||
|
if "query_full" in query_params:
|
||||||
|
if query_params["query_full"] == "":
|
||||||
|
del query_params["query_full"]
|
||||||
|
|
||||||
|
if "tags" in query_params:
|
||||||
|
if query_params["tags"] == "":
|
||||||
|
del query_params["tags"]
|
||||||
else:
|
else:
|
||||||
context = {"object_list": []}
|
tags = parse_tags(query_params["tags"])
|
||||||
|
extra_params["tags"] = tags
|
||||||
|
|
||||||
|
context = query_results(request, query_params, **extra_params)
|
||||||
|
|
||||||
# Valid sizes
|
# Valid sizes
|
||||||
context["sizes"] = sizes
|
context["sizes"] = sizes
|
||||||
|
|
||||||
# URI we're passing to the template for linking
|
|
||||||
if "csrfmiddlewaretoken" in query_params:
|
|
||||||
del query_params["csrfmiddlewaretoken"]
|
|
||||||
|
|
||||||
url_params = urllib.parse.urlencode(query_params)
|
url_params = urllib.parse.urlencode(query_params)
|
||||||
context["client_uri"] = url_params
|
context["client_uri"] = url_params
|
||||||
|
|
||||||
context["params"] = query_params
|
context["params"] = query_params
|
||||||
if "message" in context:
|
if "message" in context:
|
||||||
response = render(request, template_name, context)
|
response = render(request, template_name, context)
|
||||||
|
@ -158,6 +189,17 @@ def drilldown_search(request, return_context=False, template=None):
|
||||||
clean_url_params = urllib.parse.urlencode(clean_params)
|
clean_url_params = urllib.parse.urlencode(clean_params)
|
||||||
context["uri"] = clean_url_params
|
context["uri"] = clean_url_params
|
||||||
|
|
||||||
|
# Warn users trying to use query string that the simple query supersedes it
|
||||||
|
if all([x in query_params for x in ["query", "query_full"]]):
|
||||||
|
context["message"] = (
|
||||||
|
"You are searching with both query types. "
|
||||||
|
"The simple query will be used. "
|
||||||
|
"The full query will be ignored. "
|
||||||
|
"Remove the text from the simple query if you wish "
|
||||||
|
"to use the full query."
|
||||||
|
)
|
||||||
|
context["class"] = "warning"
|
||||||
|
|
||||||
response = render(request, template_name, context)
|
response = render(request, template_name, context)
|
||||||
if request.GET:
|
if request.GET:
|
||||||
if request.htmx:
|
if request.htmx:
|
||||||
|
@ -260,7 +302,7 @@ class DrilldownContextModal(APIView):
|
||||||
# Lookup the hash values but don't disclose them to the user
|
# Lookup the hash values but don't disclose them to the user
|
||||||
if settings.HASHING:
|
if settings.HASHING:
|
||||||
SAFE_PARAMS = deepcopy(query_params)
|
SAFE_PARAMS = deepcopy(query_params)
|
||||||
hash_lookup(SAFE_PARAMS)
|
hash_lookup(request.user, SAFE_PARAMS)
|
||||||
else:
|
else:
|
||||||
SAFE_PARAMS = query_params
|
SAFE_PARAMS = query_params
|
||||||
|
|
||||||
|
@ -383,7 +425,7 @@ class ThresholdInfoModal(APIView):
|
||||||
# Lookup the hash values but don't disclose them to the user
|
# Lookup the hash values but don't disclose them to the user
|
||||||
if settings.HASHING:
|
if settings.HASHING:
|
||||||
SAFE_PARAMS = request.data.dict()
|
SAFE_PARAMS = request.data.dict()
|
||||||
hash_lookup(SAFE_PARAMS)
|
hash_lookup(request.user, SAFE_PARAMS)
|
||||||
safe_net = SAFE_PARAMS["net"]
|
safe_net = SAFE_PARAMS["net"]
|
||||||
safe_nick = SAFE_PARAMS["nick"]
|
safe_nick = SAFE_PARAMS["nick"]
|
||||||
safe_channel = SAFE_PARAMS["channel"]
|
safe_channel = SAFE_PARAMS["channel"]
|
||||||
|
|
Loading…
Reference in New Issue