Switch to UWSGI and improve Docker definitions

.pre-commit-config.yaml

@@ -24,8 +24,7 @@ repos:
         exclude : ^core/static/css # slow
       - id: djjs
         exclude: ^core/static/js # slow
-#  - repo: https://github.com/thibaudcolas/curlylint
+  - repo: https://github.com/sirwart/ripsecrets.git
-#    rev: v0.13.1
+    rev: v0.1.5
-#    hooks:
+    hooks:
-#      - id: curlylint
+      - id: ripsecrets
-#        files: \.(html|sls)$

Dockerfile

@@ -0,0 +1,28 @@
+# syntax=docker/dockerfile:1
+FROM python:3
+ARG OPERATION
+
+RUN useradd -d /code pathogen
+RUN mkdir -p /code
+RUN chown -R pathogen:pathogen /code
+
+RUN mkdir -p /conf/static
+RUN chown -R pathogen:pathogen /conf
+
+RUN mkdir /venv
+RUN chown pathogen:pathogen /venv
+
+USER pathogen
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
+WORKDIR /code
+COPY requirements.txt /code/
+RUN python -m venv /venv
+RUN . /venv/bin/activate && pip install -r requirements.txt
+
+# CMD . /venv/bin/activate && uwsgi --ini /conf/uwsgi.ini
+
+CMD if [ "$OPERATION" = "uwsgi" ] ; then . /venv/bin/activate && uwsgi --ini /conf/uwsgi.ini ; else . /venv/bin/activate && exec python manage.py runserver 0.0.0.0:8000; fi
+
+# CMD . /venv/bin/activate && uvicorn --reload --reload-include *.html --workers 2 --uds /var/run/socks/app.sock app.asgi:application
+# CMD . /venv/bin/activate && gunicorn -b 0.0.0.0:8000 --reload app.asgi:application -k uvicorn.workers.UvicornWorker

Makefile

@@ -0,0 +1,20 @@
+run:
+	docker-compose --env-file=stack.env up -d
+
+build:
+	docker-compose --env-file=stack.env build
+
+stop:
+	docker-compose --env-file=stack.env down
+
+log:
+	docker-compose --env-file=stack.env logs -f
+
+migrate:
+	docker-compose --env-file=stack.env run --rm app sh -c ". /venv/bin/activate && python manage.py migrate"
+
+makemigrations:
+	docker-compose --env-file=stack.env run --rm app sh -c ". /venv/bin/activate && python manage.py makemigrations"
+
+auth:
+	docker-compose --env-file=stack.env run --rm app sh -c ". /venv/bin/activate && python manage.py createsuperuser"

app/local_settings.example.py

@@ -1,40 +1,37 @@
-# OpenSearch settings
+# Elasticsearch settings
-OPENSEARCH_URL = "127.0.0.1"
+ELASTICSEARCH_URL = "10.1.0.1"
-OPENSEARCH_PORT = 9200
+ELASTICSEARCH_PORT = 9200
-OPENSEARCH_TLS = True
+ELASTICSEARCH_TLS = True
-OPENSEARCH_USERNAME = "admin"
+ELASTICSEARCH_USERNAME = "admin"
-OPENSEARCH_PASSWORD = ""
+ELASTICSEARCH_PASSWORD = "secret"
-
-OPENSEARCH_INDEX_MAIN = "pathogen-main"
-OPENSEARCH_INDEX_META = "pathogen-meta"
-OPENSEARCH_INDEX_INT = "pathogen-int"
-
-OPENSEARCH_MAIN_SIZES = ["20", "50", "100", "200", "400", "800"]
-OPENSEARCH_MAIN_SIZES_ANON = ["20", "50", "100"]
-OPENSEARCH_MAIN_SOURCES = ["dis", "4ch", "all"]
-OPENSEARCH_SOURCES_RESTRICTED = ["irc"]
 
 # Manticore settings
-MANTICORE_URL = "http://monolith-db-1:9308"
+MANTICORE_URL = "http://example-db-1:9308"
-MANTICORE_INDEX_MAIN = "main"
-MANTICORE_INDEX_META = "meta"
-MANTICORE_INDEX_INT = "internal"
 
-MANTICORE_MAIN_SIZES = ["20", "50", "100", "200", "400", "800"]
+DB_BACKEND = "ELASTICSEARCH"
-MANTICORE_MAIN_SIZES_ANON = ["20", "50", "100"]
+
-MANTICORE_MAIN_SOURCES = ["dis", "4ch", "all"]
+# Common DB settings
-MANTICORE_SOURCES_RESTRICTED = ["irc"]
+INDEX_MAIN = "main"
-MANTICORE_CACHE = True
+INDEX_RESTRICTED = "restricted"
-MANTICORE_CACHE_TIMEOUT = 60
+INDEX_META = "meta"
+INDEX_INT = "internal"
+
+MAIN_SIZES = ["1", "5", "15", "30", "50", "100", "250", "500", "1000"]
+MAIN_SIZES_ANON = ["1", "5", "15", "30", "50", "100"]
+MAIN_SOURCES = ["dis", "4ch", "all"]
+SOURCES_RESTRICTED = ["irc"]
+CACHE = False
+CACHE_TIMEOUT = 2
 
 DRILLDOWN_RESULTS_PER_PAGE = 15
 DRILLDOWN_DEFAULT_PARAMS = {
-    "size": "20",
+    "size": "15",
     "index": "main",
     "sorting": "desc",
     "source": "4ch",
 }
 
+
 # Encryption
 # ENCRYPTION = False
 # ENCRYPTION_KEY = b""
@@ -61,7 +58,7 @@ DRILLDOWN_DEFAULT_PARAMS = {
 # # Delay results by this many days
 # DELAY_DURATION = 10
 
-OPENSEARCH_BLACKLISTED = {}
+ELASTICSEARCH_BLACKLISTED = {}
 
 
 # URLs\
@@ -89,8 +86,8 @@ SECRET_KEY = "a"
 STRIPE_ADMIN_COUPON = ""
 
 # Threshold
-THRESHOLD_ENDPOINT = "http://threshold-app-1:13869"
+THRESHOLD_ENDPOINT = "http://threshold:13869"
-THRESHOLD_API_KEY = ""
+THRESHOLD_API_KEY = "api_1"
 THRESHOLD_API_TOKEN = ""
 THRESHOLD_API_COUNTER = ""
 
@@ -106,12 +103,3 @@ META_QUERY_SIZE = 10000
 
 DEBUG = True
 PROFILER = False
-
-if DEBUG:
-    import socket  # only if you haven't already imported this
-
-    hostname, _, ips = socket.gethostbyname_ex(socket.gethostname())
-    INTERNAL_IPS = [ip[: ip.rfind(".")] + ".1" for ip in ips] + [
-        "127.0.0.1",
-        "10.0.2.2",
-    ]

core/__init__.py

@@ -2,7 +2,7 @@ import stripe
 from django.conf import settings
 from redis import StrictRedis
 
-r = StrictRedis(unix_socket_path="/var/run/redis/redis.sock", db=0)
+r = StrictRedis(unix_socket_path="/var/run/socks/redis.sock", db=0)
 
 if settings.STRIPE_TEST:
     stripe.api_key = settings.STRIPE_API_KEY_TEST

core/db/__init__.py

@@ -1,6 +1,7 @@
 import random
 import string
 import time
+from abc import ABC, abstractmethod
 from datetime import datetime
 from math import floor, log10
 
@@ -14,7 +15,7 @@ from core.util import logs
 from core.views import helpers
 
 
-class StorageBackend(object):
+class StorageBackend(ABC):
     def __init__(self, name):
         self.log = logs.get_logger(name)
         self.log.info(f"Initialising storage backend {name}")
@@ -22,8 +23,9 @@ class StorageBackend(object):
         self.initialise_caching()
         self.initialise()
 
+    @abstractmethod
     def initialise(self, **kwargs):
-        raise NotImplementedError
+        pass
 
     def initialise_caching(self):
         hash_key = r.get("cache_hash_key")
@@ -37,11 +39,13 @@ class StorageBackend(object):
             self.log.debug(f"Decoded hash key: {hash_key}")
         self.hash_key = hash_key
 
+    @abstractmethod
     def construct_query(self, **kwargs):
-        raise NotImplementedError
+        pass
 
+    @abstractmethod
     def run_query(self, **kwargs):
-        raise NotImplementedError
+        pass
 
     def parse_size(self, query_params, sizes):
         if "size" in query_params:
@@ -93,22 +97,22 @@ class StorageBackend(object):
             index = settings.INDEX_MAIN
         return index
 
-    def parse_query(self, query_params, tags, size, index, custom_query, add_bool):
+    def parse_query(self, query_params, tags, size, custom_query, add_bool, **kwargs):
         query_created = False
         if "query" in query_params:
             query = query_params["query"]
-            search_query = self.construct_query(query, size, index)
+            search_query = self.construct_query(query, size, **kwargs)
             query_created = True
         else:
             if custom_query:
                 search_query = custom_query
             else:
-                search_query = self.construct_query(None, size, index, blank=True)
+                search_query = self.construct_query(None, size, blank=True, **kwargs)
 
         if tags:
             # Get a blank search query
             if not query_created:
-                search_query = self.construct_query(None, size, index, blank=True)
+                search_query = self.construct_query(None, size, blank=True, **kwargs)
                 query_created = True
             for item in tags:
                 for tagname, tagvalue in item.items():
@@ -217,7 +221,7 @@ class StorageBackend(object):
         # For every hit from ES
         for index, item in enumerate(list(response["hits"]["hits"])):
             # For every blacklisted type
-            for blacklisted_type in settings.OPENSEARCH_BLACKLISTED.keys():
+            for blacklisted_type in settings.ELASTICSEARCH_BLACKLISTED.keys():
                 # Check this field we are matching exists
                 if "_source" in item.keys():
                     data_index = "_source"
@@ -228,9 +232,7 @@ class StorageBackend(object):
                 if blacklisted_type in item[data_index].keys():
                     content = item[data_index][blacklisted_type]
                     # For every item in the blacklisted array for the type
-                    for blacklisted_item in settings.OPENSEARCH_BLACKLISTED[
+                    for blacklisted_item in settings.BLACKLISTED[blacklisted_type]:
-                        blacklisted_type
-                    ]:
                         if blacklisted_item == str(content):
                             # Remove the item
                             if item in response["hits"]["hits"]:
@@ -255,7 +257,7 @@ class StorageBackend(object):
         # Actually get rid of all the things we set to None
         response["hits"]["hits"] = [hit for hit in response["hits"]["hits"] if hit]
 
-    def query(self, user, search_query):
+    def query(self, user, search_query, **kwargs):
         # For time tracking
         start = time.process_time()
         if settings.CACHE:
@@ -265,8 +267,6 @@ class StorageBackend(object):
             cache_hit = r.get(f"query_cache.{user.id}.{hash}")
             if cache_hit:
                 response = orjson.loads(cache_hit)
-                print("CACHE HIT", response)
-
                 time_took = (time.process_time() - start) * 1000
                 # Round to 3 significant figures
                 time_took_rounded = round(
@@ -277,7 +277,19 @@ class StorageBackend(object):
                     "took": time_took_rounded,
                     "cache": True,
                 }
-        response = self.run_query(user, search_query)
+        response = self.run_query(user, search_query, **kwargs)
+
+        # For Elasticsearch
+        if isinstance(response, Exception):
+            message = f"Error: {response.info['error']['root_cause'][0]['type']}"
+            message_class = "danger"
+            return {"message": message, "class": message_class}
+        if len(response["hits"]["hits"]) == 0:
+            message = "No results."
+            message_class = "danger"
+            return {"message": message, "class": message_class}
+
+        # For Druid
         if "error" in response:
             if "errorMessage" in response:
                 context = {
@@ -287,12 +299,12 @@ class StorageBackend(object):
                 return context
             else:
                 return response
-        # response = response.to_dict()
-        # print("RESP", response)
         if "took" in response:
             if response["took"] is None:
                 return None
-        self.filter_blacklisted(user, response)
+
+        # Removed for now, no point given we have restricted indexes
+        # self.filter_blacklisted(user, response)
 
         # Parse the response
         response_parsed = self.parse(response)
@@ -308,18 +320,20 @@ class StorageBackend(object):
         time_took_rounded = round(time_took, 3 - int(floor(log10(abs(time_took)))) - 1)
         return {"object_list": response_parsed, "took": time_took_rounded}
 
+    @abstractmethod
     def query_results(self, **kwargs):
-        raise NotImplementedError
+        pass
 
     def process_results(self, response, **kwargs):
         if kwargs.get("annotate"):
             annotate_results(response)
-        if kwargs.get("dedup"):
+        if kwargs.get("reverse"):
-            response = response[::-1]
+            response.reverse()
         if kwargs.get("dedup"):
             if not kwargs.get("dedup_fields"):
                 dedup_fields = ["msg", "nick", "ident", "host", "net", "channel"]
             response = helpers.dedup_list(response, dedup_fields)
 
+    @abstractmethod
     def parse(self, response):
-        raise NotImplementedError
+        pass

core/db/druid.py

@@ -77,7 +77,8 @@ class DruidBackend(StorageBackend):
             self.add_type("or", search_query, extra_should2)
         return search_query
 
-    def construct_query(self, query, size, index, blank=False):
+    def construct_query(self, query, size, blank=False, **kwargs):
+        index = kwargs.get("index")
         search_query = {
             "limit": size,
             "queryType": "scan",
@@ -107,19 +108,13 @@ class DruidBackend(StorageBackend):
 
     def parse(self, response):
         parsed = parse_druid(response)
-        print("PARSE LEN", len(parsed))
         return parsed
 
     def run_query(self, user, search_query):
         ss = orjson.dumps(search_query, option=orjson.OPT_INDENT_2)
         ss = ss.decode()
-        print(ss)
+        response = requests.post("http://druid:8082/druid/v2", json=search_query)
-        response = requests.post("http://broker:8082/druid/v2", json=search_query)
         response = orjson.loads(response.text)
-        print("RESPONSE LEN", len(response))
-        # ss = orjson.dumps(response, option=orjson.OPT_INDENT_2)
-        # ss = ss.decode()
-        # print(ss)
         return response
 
     def filter_blacklisted(self, user, response):
@@ -172,7 +167,7 @@ class DruidBackend(StorageBackend):
 
         # Q/T - Query/Tags
         search_query = self.parse_query(
-            query_params, tags, size, index, custom_query, add_bool
+            query_params, tags, size, custom_query, add_bool, index=index
         )
         # Query should be a dict, so check if it contains message here
         if "message" in search_query:
@@ -239,11 +234,6 @@ class DruidBackend(StorageBackend):
             dedup_fields=dedup_fields,
             reverse=reverse,
         )
-        # ss = orjson.dumps(list(response), option=orjson.OPT_INDENT_2)
-        # ss = ss.decode()
-        # print(ss)
-        # print("PARSED", results_parsed)
-        # return results_parsed
         context = response
         return context
 

core/db/elastic.py

@@ -0,0 +1,375 @@
+# from copy import deepcopy
+# from datetime import datetime, timedelta
+
+from django.conf import settings
+from elasticsearch import Elasticsearch
+from elasticsearch.exceptions import NotFoundError, RequestError
+
+from core.db import StorageBackend
+
+# from json import dumps
+# pp = lambda x: print(dumps(x, indent=2))
+from core.db.processing import parse_results
+from core.views import helpers
+
+
+class ElasticsearchBackend(StorageBackend):
+    def __init__(self):
+        super().__init__("Elasticsearch")
+
+    def initialise(self, **kwargs):
+        """
+        Inititialise the Elastuicsearch API endpoint.
+        """
+        auth = (settings.ELASTICSEARCH_USERNAME, settings.ELASTICSEARCH_PASSWORD)
+        client = Elasticsearch(
+            settings.ELASTICSEARCH_URL, http_auth=auth, verify_certs=False
+        )
+        self.client = client
+
+    def construct_context_query(
+        self, index, net, channel, src, num, size, type=None, nicks=None
+    ):
+        # Get the initial query
+        query = self.construct_query(None, size, blank=True)
+
+        extra_must = []
+        extra_should = []
+        extra_should2 = []
+        if num:
+            extra_must.append({"match_phrase": {"num": num}})
+        if net:
+            extra_must.append({"match_phrase": {"net": net}})
+        if channel:
+            extra_must.append({"match": {"channel": channel}})
+        if nicks:
+            for nick in nicks:
+                extra_should2.append({"match": {"nick": nick}})
+
+        types = ["msg", "notice", "action", "kick", "topic", "mode"]
+        fields = [
+            "nick",
+            "ident",
+            "host",
+            "channel",
+            "ts",
+            "msg",
+            "type",
+            "net",
+            "src",
+            "tokens",
+        ]
+        query["fields"] = fields
+
+        if index == "internal":
+            fields.append("mtype")
+            if channel == "*status" or type == "znc":
+                if {"match": {"channel": channel}} in extra_must:
+                    extra_must.remove({"match": {"channel": channel}})
+                extra_should2 = []
+                # Type is one of msg or notice
+                # extra_should.append({"match": {"mtype": "msg"}})
+                # extra_should.append({"match": {"mtype": "notice"}})
+                extra_should.append({"match": {"type": "znc"}})
+                extra_should.append({"match": {"type": "self"}})
+
+                extra_should2.append({"match": {"type": "znc"}})
+                extra_should2.append({"match": {"nick": channel}})
+            elif type == "auth":
+                if {"match": {"channel": channel}} in extra_must:
+                    extra_must.remove({"match": {"channel": channel}})
+                extra_should2 = []
+                extra_should2.append({"match": {"nick": channel}})
+                # extra_should2.append({"match": {"mtype": "msg"}})
+                # extra_should2.append({"match": {"mtype": "notice"}})
+
+                extra_should.append({"match": {"type": "query"}})
+                extra_should2.append({"match": {"type": "self"}})
+                extra_should.append({"match": {"nick": channel}})
+            else:
+                for ctype in types:
+                    extra_should.append({"equals": {"mtype": ctype}})
+        else:
+            for ctype in types:
+                extra_should.append({"match": {"type": ctype}})
+        # query = {
+        #     "index": index,
+        #     "limit": size,
+        #     "query": {
+        #         "bool": {
+        #             "must": [
+        #                 # {"equals": {"src": src}},
+        #                 # {
+        #                 #     "bool": {
+        #                 #         "should": [*extra_should],
+        #                 #     }
+        #                 # },
+        #                 # {
+        #                 #     "bool": {
+        #                 #         "should": [*extra_should2],
+        #                 #     }
+        #                 # },
+        #                 *extra_must,
+        #             ]
+        #         }
+        #     },
+        #     "fields": fields,
+        #     # "_source": False,
+        # }
+        if extra_must:
+            for x in extra_must:
+                query["query"]["bool"]["must"].append(x)
+        if extra_should:
+            query["query"]["bool"]["must"].append({"bool": {"should": [*extra_should]}})
+        if extra_should2:
+            query["query"]["bool"]["must"].append(
+                {"bool": {"should": [*extra_should2]}}
+            )
+        return query
+
+    def construct_query(self, query, size, blank=False):
+        """
+        Accept some query parameters and construct an Elasticsearch query.
+        """
+        query_base = {
+            "size": size,
+            "query": {"bool": {"must": []}},
+        }
+        query_string = {
+            "query_string": {
+                "query": query,
+                # "fields": fields,
+                # "default_field": "msg",
+                # "type": "best_fields",
+                "fuzziness": "AUTO",
+                "fuzzy_transpositions": True,
+                "fuzzy_max_expansions": 50,
+                "fuzzy_prefix_length": 0,
+                # "minimum_should_match": 1,
+                "default_operator": "or",
+                "analyzer": "standard",
+                "lenient": True,
+                "boost": 1,
+                "allow_leading_wildcard": True,
+                # "enable_position_increments": False,
+                "phrase_slop": 3,
+                # "max_determinized_states": 10000,
+                "quote_field_suffix": "",
+                "quote_analyzer": "standard",
+                "analyze_wildcard": False,
+                "auto_generate_synonyms_phrase_query": True,
+            }
+        }
+        if not blank:
+            query_base["query"]["bool"]["must"].append(query_string)
+        return query_base
+
+    def parse(self, response):
+        parsed = parse_results(response)
+        return parsed
+
+    def run_query(self, user, search_query, **kwargs):
+        """
+        Low level helper to run an ES query.
+        Accept a user to pass it to the filter, so we can
+        avoid filtering for superusers.
+        Accept fields and size, for the fields we want to match and the
+        number of results to return.
+        """
+        index = kwargs.get("index")
+        try:
+            response = self.client.search(body=search_query, index=index)
+        except RequestError as err:
+            print("Elasticsearch error", err)
+            return err
+        except NotFoundError as err:
+            print("Elasticsearch error", err)
+            return err
+        return response
+
+    def query_results(
+        self,
+        request,
+        query_params,
+        size=None,
+        annotate=True,
+        custom_query=False,
+        reverse=False,
+        dedup=False,
+        dedup_fields=None,
+        tags=None,
+    ):
+
+        add_bool = []
+        add_top = []
+        add_top_negative = []
+
+        helpers.add_defaults(query_params)
+
+        # Now, run the helpers for SIQTSRSS/ADR
+        # S - Size
+        # I - Index
+        # Q - Query
+        # T - Tags
+        # S - Source
+        # R - Ranges
+        # S - Sort
+        # S - Sentiment
+        # A - Annotate
+        # D - Dedup
+        # R - Reverse
+
+        # S - Size
+        if request.user.is_anonymous:
+            sizes = settings.MAIN_SIZES_ANON
+        else:
+            sizes = settings.MAIN_SIZES
+        if not size:
+            size = self.parse_size(query_params, sizes)
+            if isinstance(size, dict):
+                return size
+
+        # I - Index
+        index = self.parse_index(request.user, query_params)
+        if isinstance(index, dict):
+            return index
+
+        # Q/T - Query/Tags
+        search_query = self.parse_query(
+            query_params, tags, size, custom_query, add_bool
+        )
+        # Query should be a dict, so check if it contains message here
+        if "message" in search_query:
+            return search_query
+
+        # S - Sources
+        sources = self.parse_source(request.user, query_params)
+        if isinstance(sources, dict):
+            return sources
+        total_count = len(sources)
+        total_sources = len(settings.MAIN_SOURCES) + len(settings.SOURCES_RESTRICTED)
+        if total_count != total_sources:
+            add_top_tmp = {"bool": {"should": []}}
+            for source_iter in sources:
+                add_top_tmp["bool"]["should"].append(
+                    {"match_phrase": {"src": source_iter}}
+                )
+            add_top.append(add_top_tmp)
+
+        # R - Ranges
+        # date_query = False
+        from_ts, to_ts = self.parse_date_time(query_params)
+        if from_ts:
+            range_query = {
+                "range": {
+                    "ts": {
+                        "gt": from_ts,
+                        "lt": to_ts,
+                    }
+                }
+            }
+            add_top.append(range_query)
+
+        # S - Sort
+        sort = self.parse_sort(query_params)
+        if isinstance(sort, dict):
+            return sort
+        if sort:
+            # For Druid compatibility
+            sort_map = {"ascending": "asc", "descending": "desc"}
+            sorting = [
+                {
+                    "ts": {
+                        "order": sort_map[sort],
+                    }
+                }
+            ]
+            search_query["sort"] = sorting
+
+        # S - Sentiment
+        sentiment_r = self.parse_sentiment(query_params)
+        if isinstance(sentiment_r, dict):
+            return sentiment_r
+        if sentiment_r:
+            sentiment_method, sentiment = sentiment_r
+            range_query_compare = {"range": {"sentiment": {}}}
+            range_query_precise = {
+                "match": {
+                    "sentiment": None,
+                }
+            }
+            if sentiment_method == "below":
+                range_query_compare["range"]["sentiment"]["lt"] = sentiment
+                add_top.append(range_query_compare)
+            elif sentiment_method == "above":
+                range_query_compare["range"]["sentiment"]["gt"] = sentiment
+                add_top.append(range_query_compare)
+            elif sentiment_method == "exact":
+                range_query_precise["match"]["sentiment"] = sentiment
+                add_top.append(range_query_precise)
+            elif sentiment_method == "nonzero":
+                range_query_precise["match"]["sentiment"] = 0
+                add_top_negative.append(range_query_precise)
+
+        # Add in the additional information we already populated
+        self.add_bool(search_query, add_bool)
+        self.add_top(search_query, add_top)
+        self.add_top(search_query, add_top_negative, negative=True)
+
+        response = self.query(
+            request.user,
+            search_query,
+            index=index,
+        )
+        if "message" in response:
+            return response
+
+        # A/D/R - Annotate/Dedup/Reverse
+        self.process_results(
+            response["object_list"],
+            annotate=annotate,
+            dedup=dedup,
+            dedup_fields=dedup_fields,
+            reverse=reverse,
+        )
+
+        context = response
+        return context
+
+    def query_single_result(self, request, query_params):
+        context = self.query_results(request, query_params, size=100)
+
+        if not context:
+            return {"message": "Failed to run query", "message_class": "danger"}
+        if "message" in context:
+            return context
+        dedup_set = {item["nick"] for item in context["object_list"]}
+        if dedup_set:
+            context["item"] = context["object_list"][0]
+
+        return context
+
+    def add_bool(self, search_query, add_bool):
+        """
+        Add the specified boolean matches to search query.
+        """
+        if not add_bool:
+            return
+        for item in add_bool:
+            search_query["query"]["bool"]["must"].append({"match_phrase": item})
+
+    def add_top(self, search_query, add_top, negative=False):
+        """
+        Merge add_top with the base of the search_query.
+        """
+        if not add_top:
+            return
+        if negative:
+            for item in add_top:
+                if "must_not" in search_query["query"]["bool"]:
+                    search_query["query"]["bool"]["must_not"].append(item)
+                else:
+                    search_query["query"]["bool"]["must_not"] = [item]
+        else:
+            for item in add_top:
+                search_query["query"]["bool"]["must"].append(item)

core/db/opensearch.py

@@ -1,485 +0,0 @@
-# from copy import deepcopy
-# from datetime import datetime, timedelta
-
-from django.conf import settings
-from opensearchpy import OpenSearch
-from opensearchpy.exceptions import NotFoundError, RequestError
-
-from core.db import StorageBackend
-
-# from json import dumps
-# pp = lambda x: print(dumps(x, indent=2))
-from core.db.processing import annotate_results, parse_results
-from core.views.helpers import dedup_list
-
-
-class OpensearchBackend(StorageBackend):
-    def __init__(self):
-        super().__init__("Opensearch")
-
-    def initialise(self, **kwargs):
-        """
-        Inititialise the OpenSearch API endpoint.
-        """
-        auth = (settings.OPENSEARCH_USERNAME, settings.OPENSEARCH_PASSWORD)
-        client = OpenSearch(
-            # fmt: off
-            hosts=[{"host": settings.OPENSEARCH_URL,
-                    "port": settings.OPENSEARCH_PORT}],
-            http_compress=False,  # enables gzip compression for request bodies
-            http_auth=auth,
-            # client_cert = client_cert_path,
-            # client_key = client_key_path,
-            use_ssl=settings.OPENSEARCH_TLS,
-            verify_certs=False,
-            ssl_assert_hostname=False,
-            ssl_show_warn=False,
-            # a_certs=ca_certs_path,
-        )
-        self.client = client
-
-    def construct_query(self, query, size, use_query_string=True, tokens=False):
-        """
-        Accept some query parameters and construct an OpenSearch query.
-        """
-        if not size:
-            size = 5
-        query_base = {
-            "size": size,
-            "query": {"bool": {"must": []}},
-        }
-        query_string = {
-            "query_string": {
-                "query": query,
-                # "fields": fields,
-                # "default_field": "msg",
-                # "type": "best_fields",
-                "fuzziness": "AUTO",
-                "fuzzy_transpositions": True,
-                "fuzzy_max_expansions": 50,
-                "fuzzy_prefix_length": 0,
-                # "minimum_should_match": 1,
-                "default_operator": "or",
-                "analyzer": "standard",
-                "lenient": True,
-                "boost": 1,
-                "allow_leading_wildcard": True,
-                # "enable_position_increments": False,
-                "phrase_slop": 3,
-                # "max_determinized_states": 10000,
-                "quote_field_suffix": "",
-                "quote_analyzer": "standard",
-                "analyze_wildcard": False,
-                "auto_generate_synonyms_phrase_query": True,
-            }
-        }
-        query_tokens = {
-            "simple_query_string": {
-                # "tokens": query,
-                "query": query,
-                "fields": ["tokens"],
-                "flags": "ALL",
-                "fuzzy_transpositions": True,
-                "fuzzy_max_expansions": 50,
-                "fuzzy_prefix_length": 0,
-                "default_operator": "and",
-                "analyzer": "standard",
-                "lenient": True,
-                "boost": 1,
-                "quote_field_suffix": "",
-                "analyze_wildcard": False,
-                "auto_generate_synonyms_phrase_query": False,
-            }
-        }
-        if tokens:
-            query_base["query"]["bool"]["must"].append(query_tokens)
-            # query["query"]["bool"]["must"].append(query_string)
-            # query["query"]["bool"]["must"][0]["query_string"]["fields"] = ["tokens"]
-        elif use_query_string:
-            query_base["query"]["bool"]["must"].append(query_string)
-        return query_base
-
-    def run_query(self, client, user, query, custom_query=False, index=None, size=None):
-        """
-        Low level helper to run an ES query.
-        Accept a user to pass it to the filter, so we can
-        avoid filtering for superusers.
-        Accept fields and size, for the fields we want to match and the
-        number of results to return.
-        """
-        if not index:
-            index = settings.INDEX_MAIN
-        if custom_query:
-            search_query = query
-        else:
-            search_query = self.construct_query(query, size)
-        try:
-            response = client.search(body=search_query, index=index)
-        except RequestError as err:
-            print("OpenSearch error", err)
-            return err
-        except NotFoundError as err:
-            print("OpenSearch error", err)
-            return err
-        return response
-
-    def query_results(
-        self,
-        request,
-        query_params,
-        size=None,
-        annotate=True,
-        custom_query=False,
-        reverse=False,
-        dedup=False,
-        dedup_fields=None,
-        lookup_hashes=True,
-        tags=None,
-    ):
-        """
-        API helper to alter the OpenSearch return format into something
-        a bit better to parse.
-        Accept a HTTP request object. Run the query, and annotate the
-        results with the other data we have.
-        """
-        # is_anonymous = isinstance(request.user, AnonymousUser)
-        query = None
-        message = None
-        message_class = None
-        add_bool = []
-        add_top = []
-        add_top_negative = []
-        sort = None
-        query_created = False
-
-        # Lookup the hash values but don't disclose them to the user
-        # denied = []
-        # if lookup_hashes:
-        #     if settings.HASHING:
-        #         query_params = deepcopy(query_params)
-        #         denied_q = hash_lookup(request.user, query_params)
-        #         denied.extend(denied_q)
-        #         if tags:
-        #             denied_t = hash_lookup(request.user, tags, query_params)
-        #             denied.extend(denied_t)
-
-        # message = "Permission denied: "
-        # for x in denied:
-        #     if isinstance(x, SearchDenied):
-        #         message += f"Search({x.key}: {x.value}) "
-        #     elif isinstance(x, LookupDenied):
-        #         message += f"Lookup({x.key}: {x.value}) "
-        # if denied:
-        #     # message = [f"{i}" for i in message]
-        #     # message = "\n".join(message)
-        #     message_class = "danger"
-        #     return {"message": message, "class": message_class}
-
-        if request.user.is_anonymous:
-            sizes = settings.MAIN_SIZES_ANON
-        else:
-            sizes = settings.MAIN_SIZES
-        if not size:
-            if "size" in query_params:
-                size = query_params["size"]
-                if size not in sizes:
-                    message = "Size is not permitted"
-                    message_class = "danger"
-                    return {"message": message, "class": message_class}
-            else:
-                size = 20
-        source = None
-        if "source" in query_params:
-            source = query_params["source"]
-
-            if source in settings.SOURCES_RESTRICTED:
-                if not request.user.has_perm("core.restricted_sources"):
-                    message = "Access denied"
-                    message_class = "danger"
-                    return {"message": message, "class": message_class}
-            elif source not in settings.MAIN_SOURCES:
-                message = "Invalid source"
-                message_class = "danger"
-                return {"message": message, "class": message_class}
-
-            if source == "all":
-                source = None  # the next block will populate it
-
-        if source:
-            sources = [source]
-        else:
-            sources = settings.MAIN_SOURCES
-            if request.user.has_perm("core.restricted_sources"):
-                for source_iter in settings.SOURCES_RESTRICTED:
-                    sources.append(source_iter)
-
-        add_top_tmp = {"bool": {"should": []}}
-        for source_iter in sources:
-            add_top_tmp["bool"]["should"].append({"match_phrase": {"src": source_iter}})
-        add_top.append(add_top_tmp)
-
-        # date_query = False
-        if set({"from_date", "to_date", "from_time", "to_time"}).issubset(
-            query_params.keys()
-        ):
-            from_ts = f"{query_params['from_date']}T{query_params['from_time']}Z"
-            to_ts = f"{query_params['to_date']}T{query_params['to_time']}Z"
-            range_query = {
-                "range": {
-                    "ts": {
-                        "gt": from_ts,
-                        "lt": to_ts,
-                    }
-                }
-            }
-            add_top.append(range_query)
-
-        # if date_query:
-        #     if settings.DELAY_RESULTS:
-        #         if source not in settings.SAFE_SOURCES:
-        #             if request.user.has_perm("core.bypass_delay"):
-        #                 add_top.append(range_query)
-        #             else:
-        #                 delay_as_ts = datetime.now() - timedelta(
-        #                     days=settings.DELAY_DURATION
-        #                 )
-        #                 lt_as_ts = datetime.strptime(
-        #                     range_query["range"]["ts"]["lt"], "%Y-%m-%dT%H:%MZ"
-        #                 )
-        #                 if lt_as_ts > delay_as_ts:
-        #                     range_query["range"]["ts"][
-        #                         "lt"
-        #                     ] = f"now-{settings.DELAY_DURATION}d"
-        #                 add_top.append(range_query)
-        #     else:
-        #         add_top.append(range_query)
-        # else:
-        #     if settings.DELAY_RESULTS:
-        #         if source not in settings.SAFE_SOURCES:
-        #             if not request.user.has_perm("core.bypass_delay"):
-        #                 range_query = {
-        #                     "range": {
-        #                         "ts": {
-        #                             # "gt": ,
-        #                             "lt": f"now-{settings.DELAY_DURATION}d",
-        #                         }
-        #                     }
-        #                 }
-        #                 add_top.append(range_query)
-
-        if "sorting" in query_params:
-            sorting = query_params["sorting"]
-            if sorting not in ("asc", "desc", "none"):
-                message = "Invalid sort"
-                message_class = "danger"
-                return {"message": message, "class": message_class}
-            if sorting in ("asc", "desc"):
-                sort = [
-                    {
-                        "ts": {
-                            "order": sorting,
-                        }
-                    }
-                ]
-
-        if "check_sentiment" in query_params:
-            if "sentiment_method" not in query_params:
-                message = "No sentiment method"
-                message_class = "danger"
-                return {"message": message, "class": message_class}
-            if "sentiment" in query_params:
-                sentiment = query_params["sentiment"]
-                try:
-                    sentiment = float(sentiment)
-                except ValueError:
-                    message = "Sentiment is not a float"
-                    message_class = "danger"
-                    return {"message": message, "class": message_class}
-            sentiment_method = query_params["sentiment_method"]
-            range_query_compare = {"range": {"sentiment": {}}}
-            range_query_precise = {
-                "match": {
-                    "sentiment": None,
-                }
-            }
-            if sentiment_method == "below":
-                range_query_compare["range"]["sentiment"]["lt"] = sentiment
-                add_top.append(range_query_compare)
-            elif sentiment_method == "above":
-                range_query_compare["range"]["sentiment"]["gt"] = sentiment
-                add_top.append(range_query_compare)
-            elif sentiment_method == "exact":
-                range_query_precise["match"]["sentiment"] = sentiment
-                add_top.append(range_query_precise)
-            elif sentiment_method == "nonzero":
-                range_query_precise["match"]["sentiment"] = 0
-                add_top_negative.append(range_query_precise)
-
-        # Only one of query or query_full can be active at once
-        # We prefer query because it's simpler
-        if "query" in query_params:
-            query = query_params["query"]
-            search_query = self.construct_query(query, size, tokens=True)
-            query_created = True
-        elif "query_full" in query_params:
-            query_full = query_params["query_full"]
-            # if request.user.has_perm("core.query_search"):
-            search_query = self.construct_query(query_full, size)
-            query_created = True
-            # else:
-            #     message = "You cannot search by query string"
-            #     message_class = "danger"
-            #     return {"message": message, "class": message_class}
-        else:
-            if custom_query:
-                search_query = custom_query
-
-        if tags:
-            # Get a blank search query
-            if not query_created:
-                search_query = self.construct_query(None, size, use_query_string=False)
-                query_created = True
-            for tagname, tagvalue in tags.items():
-                add_bool.append({tagname: tagvalue})
-
-        required_any = ["query_full", "query", "tags"]
-        if not any([field in query_params.keys() for field in required_any]):
-            if not custom_query:
-                message = "Empty query!"
-                message_class = "warning"
-                return {"message": message, "class": message_class}
-
-        if add_bool:
-            # if "bool" not in search_query["query"]:
-            #     search_query["query"]["bool"] = {}
-            # if "must" not in search_query["query"]["bool"]:
-            #     search_query["query"]["bool"] = {"must": []}
-
-            for item in add_bool:
-                search_query["query"]["bool"]["must"].append({"match_phrase": item})
-        if add_top:
-            for item in add_top:
-                search_query["query"]["bool"]["must"].append(item)
-        if add_top_negative:
-            for item in add_top_negative:
-                if "must_not" in search_query["query"]["bool"]:
-                    search_query["query"]["bool"]["must_not"].append(item)
-                else:
-                    search_query["query"]["bool"]["must_not"] = [item]
-        if sort:
-            search_query["sort"] = sort
-
-        if "index" in query_params:
-            index = query_params["index"]
-            if index == "main":
-                index = settings.INDEX_MAIN
-            else:
-                if not request.user.has_perm(f"core.index_{index}"):
-                    message = "Not permitted to search by this index"
-                    message_class = "danger"
-                    return {
-                        "message": message,
-                        "class": message_class,
-                    }
-                if index == "meta":
-                    index = settings.INDEX_META
-                elif index == "internal":
-                    index = settings.INDEX_INT
-                else:
-                    message = "Index is not valid."
-                    message_class = "danger"
-                    return {
-                        "message": message,
-                        "class": message_class,
-                    }
-
-        else:
-            index = settings.INDEX_MAIN
-
-        results = self.query(
-            request.user,  # passed through run_main_query to filter_blacklisted
-            search_query,
-            custom_query=True,
-            index=index,
-            size=size,
-        )
-        if not results:
-            return False
-        if isinstance(results, Exception):
-            message = f"Error: {results.info['error']['root_cause'][0]['type']}"
-            message_class = "danger"
-            return {"message": message, "class": message_class}
-        if len(results["hits"]["hits"]) == 0:
-            message = "No results."
-            message_class = "danger"
-            return {"message": message, "class": message_class}
-
-        results_parsed = parse_results(results)
-
-        if annotate:
-            annotate_results(results_parsed)
-        if "dedup" in query_params:
-            if query_params["dedup"] == "on":
-                dedup = True
-            else:
-                dedup = False
-        else:
-            dedup = False
-
-        if reverse:
-            results_parsed = results_parsed[::-1]
-
-        if dedup:
-            if not dedup_fields:
-                dedup_fields = ["msg", "nick", "ident", "host", "net", "channel"]
-            results_parsed = dedup_list(results_parsed, dedup_fields)
-
-        # if source not in settings.SAFE_SOURCES:
-        #     if settings.ENCRYPTION:
-        #         encrypt_list(request.user, results_parsed, settings.ENCRYPTION_KEY)
-
-        #     if settings.HASHING:
-        #         hash_list(request.user, results_parsed)
-
-        #     if settings.OBFUSCATION:
-        #         obfuscate_list(request.user, results_parsed)
-
-        #     if settings.RANDOMISATION:
-        #         randomise_list(request.user, results_parsed)
-
-        # process_list(results)
-
-        # IMPORTANT! - DO NOT PASS query_params to the user!
-        context = {
-            "object_list": results_parsed,
-            "card": results["hits"]["total"]["value"],
-            "took": results["took"],
-        }
-        if "redacted" in results:
-            context["redacted"] = results["redacted"]
-        if "exemption" in results:
-            context["exemption"] = results["exemption"]
-        if query:
-            context["query"] = query
-        # if settings.DELAY_RESULTS:
-        #     if source not in settings.SAFE_SOURCES:
-        #         if not request.user.has_perm("core.bypass_delay"):
-        #             context["delay"] = settings.DELAY_DURATION
-        # if settings.RANDOMISATION:
-        #     if source not in settings.SAFE_SOURCES:
-        #         if not request.user.has_perm("core.bypass_randomisation"):
-        #             context["randomised"] = True
-        return context
-
-    def query_single_result(self, request, query_params):
-        context = self.query_results(request, query_params, size=100)
-
-        if not context:
-            return {"message": "Failed to run query", "message_class": "danger"}
-        if "message" in context:
-            return context
-        dedup_set = {item["nick"] for item in context["object_list"]}
-        if dedup_set:
-            context["item"] = context["object_list"][0]
-
-        return context

core/db/processing.py

@@ -3,7 +3,7 @@ from datetime import datetime
 from core.lib.threshold import annotate_num_chans, annotate_num_users, annotate_online
 
 
-def annotate_results(results_parsed):
+def annotate_results(results):
     """
     Accept a list of dict objects, search for the number of channels and users.
     Add them to the object.
@@ -11,7 +11,7 @@ def annotate_results(results_parsed):
     """
     # Figure out items with net (not discord)
     nets = set()
-    for x in results_parsed:
+    for x in results:
         if "net" in x:
             nets.add(x["net"])
 
@@ -21,7 +21,7 @@ def annotate_results(results_parsed):
             set(
                 [
                     x["nick"]
-                    for x in results_parsed
+                    for x in results
                     if {"nick", "src", "net"}.issubset(x)
                     and x["src"] == "irc"
                     and x["net"] == net
@@ -32,7 +32,7 @@ def annotate_results(results_parsed):
             set(
                 [
                     x["channel"]
-                    for x in results_parsed
+                    for x in results
                     if {"channel", "src", "net"}.issubset(x)
                     and x["src"] == "irc"
                     and x["net"] == net
@@ -44,7 +44,7 @@ def annotate_results(results_parsed):
         num_users = annotate_num_users(net, channels)
         # Annotate the number channels the user is on
         num_chans = annotate_num_chans(net, nicks)
-        for item in results_parsed:
+        for item in results:
             if "net" in item:
                 if item["net"] == net:
                     if "nick" in item:

core/db/storage.py

@@ -6,10 +6,10 @@ def get_db():
         from core.db.druid import DruidBackend
 
         return DruidBackend()
-    elif settings.DB_BACKEND == "OPENSEARCH":
+    elif settings.DB_BACKEND == "ELASTICSEARCH":
-        from core.db.opensearch import OpensearchBackend
+        from core.db.elastic import ElasticsearchBackend
 
-        return OpensearchBackend()
+        return ElasticsearchBackend()
     elif settings.DB_BACKEND == "MANTICORE":
         from core.db.manticore import ManticoreBackend
 

core/lib/meta.py

@@ -3,7 +3,7 @@ from math import ceil
 from django.conf import settings
 from numpy import array_split
 
-from core.db.opensearch import client, run_main_query
+from core.db.elastic import client, run_main_query
 
 
 def construct_query(net, nicks):
@@ -48,7 +48,7 @@ def get_meta(request, net, nicks, iter=True):
             request.user,
             query,
             custom_query=True,
-            index=settings.OPENSEARCH_INDEX_META,
+            index=settings.ELASTICSEARCH_INDEX_META,
         )
         if "hits" in results.keys():
             if "hits" in results["hits"]:

core/static/js/column-shifter.js

@@ -65,11 +65,12 @@ $(document).ready(function(){
                     "file_ext": "off",
                     "file_size": "off",
                     "lang_code": "off",
+                    "tokens": "off",
                     //"lang_name": "off",
-                    "words_noun": "off",
+                    // "words_noun": "off",
-                    "words_adj": "off",
+                    // "words_adj": "off",
-                    "words_verb": "off",
+                    // "words_verb": "off",
-                    "words_adv": "off"
+                    // "words_adv": "off"
                 },
             };
         } else {

core/views/ui/drilldown.py

@@ -377,6 +377,7 @@ class DrilldownContextModal(APIView):
             type=type,
             nicks=nicks_sensitive,
         )
+        print("QUERY", search_query)
         results = db.query_results(
             request,
             query_params,
@@ -389,19 +390,6 @@ class DrilldownContextModal(APIView):
         if "message" in results:
             return render(request, self.template_name, results)
 
-        # if settings.HASHING:  # we probably want to see the tokens
-        #     if query_params["source"] not in settings.SAFE_SOURCES:
-        #         if not request.user.has_perm("core.bypass_hashing"):
-        #             for index, item in enumerate(results["object_list"]):
-        #                 if "tokens" in item:
-        #                     results["object_list"][index]["msg"] = results[
-        #                         "object_list"
-        #                     ][index].pop("tokens")
-        #                     # item["msg"] = item.pop("tokens")
-
-        # Make the time nicer
-        # for index, item in enumerate(results["object_list"]):
-        #    results["object_list"][index]["time"] = item["time"]+"SSS"
         unique = str(uuid.uuid4())[:8]
         context = {
             "net": query_params["net"],

core/views/ui/tables.py

@@ -64,14 +64,13 @@ class DrilldownTable(Table):
     mtype = Column()
     realname = Column()
     server = Column()
-    mtype = Column()
+    tokens = Column()
-    # tokens = Column()
     lang_code = Column()
     lang_name = Column()
-    words_noun = Column()
+    # words_noun = Column()
-    words_adj = Column()
+    # words_adj = Column()
-    words_verb = Column()
+    # words_verb = Column()
-    words_adv = Column()
+    # words_adv = Column()
     hidden = Column()
     filename = Column()
     file_md5 = Column()

docker-compose.yml

@@ -1,17 +1,21 @@
-version: "2"
+version: "2.2"
 
 services:
   app:
     image: pathogen/neptune:latest
-    build: ./docker
+    container_name: neptune
+    build:
+      context: .
+      args:
+        OPERATION: ${OPERATION}
     volumes:
       - ${PORTAINER_GIT_DIR}:/code
-      - ${NEPTUNE_LOCAL_SETTINGS}:/code/app/local_settings.py
+      - ${PORTAINER_GIT_DIR}/docker/uwsgi.ini:/conf/uwsgi.ini
-      - ${NEPTUNE_DATABASE_FILE}:/code/db.sqlite3
+      - ${APP_LOCAL_SETTINGS}:/code/app/local_settings.py
-    ports:
+      - ${APP_DATABASE_FILE}:/code/db.sqlite3
-      - "${NEPTUNE_PORT}:8000"
+      - neptune_static:${STATIC_ROOT}
     env_file:
-      - .env
+      - stack.env
     volumes_from:
       - tmp
     depends_on:
@@ -19,49 +23,107 @@ services:
         condition: service_healthy
       migration:
         condition: service_started
+      collectstatic:
+        condition: service_started
+    networks:
+      - default
+      - pathogen
+      - elastic
 
   migration:
     image: pathogen/neptune:latest
+    container_name: migration_neptune
+    build:
+      context: .
+      args:
+        OPERATION: ${OPERATION}
     command: sh -c '. /venv/bin/activate && python manage.py migrate --noinput'
     volumes:
       - ${PORTAINER_GIT_DIR}:/code
-      - ${NEPTUNE_LOCAL_SETTINGS}:/code/app/local_settings.py
+      - ${APP_LOCAL_SETTINGS}:/code/app/local_settings.py
-      - ${NEPTUNE_DATABASE_FILE}:/code/db.sqlite3
+      - ${APP_DATABASE_FILE}:/code/db.sqlite3
+      - neptune_static:${STATIC_ROOT}
     volumes_from:
       - tmp
     depends_on:
       redis:
         condition: service_healthy
 
-  # pyroscope:
+  collectstatic:
-  #   image: pyroscope/pyroscope
+    image: pathogen/neptune:latest
-  #   environment:
+    container_name: collectstatic_neptune
-  #     - PYROSCOPE_LOG_LEVEL=debug
+    build:
-  #   ports:
+      context: .
-  #     - '4040:4040'
+      args:
-  #   command:
+        OPERATION: ${OPERATION}
-  #     - 'server'
+    command: sh -c '. /venv/bin/activate && python manage.py collectstatic --noinput'
+    volumes:
+      - ${PORTAINER_GIT_DIR}:/code
+      - ${APP_LOCAL_SETTINGS}:/code/app/local_settings.py
+      - ${APP_DATABASE_FILE}:/code/db.sqlite3
+      - neptune_static:${STATIC_ROOT}
+    env_file:
+      - stack.env
+    depends_on:
+      redis:
+        condition: service_healthy
+
+  nginx:
+    image: nginx:latest
+    container_name: nginx_neptune
+    ports:
+      - ${APP_PORT}:9999
+    ulimits:
+      nproc: 65535
+      nofile:
+         soft: 65535
+         hard: 65535
+    volumes:
+      - ${PORTAINER_GIT_DIR}:/code
+      - ${PORTAINER_GIT_DIR}/docker/nginx/conf.d/${OPERATION}.conf:/etc/nginx/conf.d/default.conf
+      - neptune_static:${STATIC_ROOT}
+    volumes_from:
+      - tmp
+    networks:
+      - default
+      - pathogen
+    depends_on:
+      app:
+        condition: service_started
 
   tmp:
     image: busybox
-    command: chmod -R 777 /var/run/redis
+    container_name: tmp_neptune
+    command: chmod -R 777 /var/run/socks
     volumes:
-      - /var/run/redis
+      - /var/run/socks
 
   redis:
     image: redis
+    container_name: redis_neptune
     command: redis-server /etc/redis.conf
+    ulimits:
+      nproc: 65535
+      nofile:
+         soft: 65535
+         hard: 65535
     volumes:
         - ${PORTAINER_GIT_DIR}/docker/redis.conf:/etc/redis.conf
     volumes_from:
       - tmp
     healthcheck:
-      test: "redis-cli -s /var/run/redis/redis.sock ping"
+      test: "redis-cli -s /var/run/socks/redis.sock ping"
       interval: 2s
       timeout: 2s
       retries: 15
 
 networks:
   default:
-      external:
+    driver: bridge
-         name: pathogen
+  pathogen:
+    external: true
+  elastic:
+    external: true
+
+volumes:
+  neptune_static: {}

docker/Dockerfile

@@ -1,18 +0,0 @@
-# syntax=docker/dockerfile:1
-FROM python:3
-
-RUN useradd -d /code pathogen
-RUN mkdir /code
-RUN chown pathogen:pathogen /code
-
-RUN mkdir /venv
-RUN chown pathogen:pathogen /venv
-
-USER pathogen
-ENV PYTHONDONTWRITEBYTECODE=1
-ENV PYTHONUNBUFFERED=1
-WORKDIR /code
-COPY requirements.dev.txt /code/
-RUN python -m venv /venv
-RUN . /venv/bin/activate && pip install -r requirements.dev.txt
-CMD . /venv/bin/activate && exec python manage.py runserver 0.0.0.0:8000

docker/docker-compose.prod.yml

@@ -1,60 +0,0 @@
-version: "2"
-   
-services:
-  app:
-    image: pathogen/neptune:latest
-    build: ./docker/prod
-    volumes:
-      - ${PORTAINER_GIT_DIR}:/code
-      - ${PORTAINER_GIT_DIR}/docker/prod/uwsgi.ini:/conf/uwsgi.ini
-      - ${NEPTUNE_LOCAL_SETTINGS}:/code/app/local_settings.py
-      - ${NEPTUNE_DATABASE_FILE}:/code/db.sqlite3
-    ports:
-      - "${NEPTUNE_PORT}:8000" # uwsgi socket
-    env_file:
-      - ../stack.env
-    volumes_from:
-      - tmp
-    depends_on:
-      redis:
-        condition: service_healthy
-      migration:
-        condition: service_started
-
-  migration:
-    image: pathogen/neptune:latest
-    build: ./docker/prod
-    command: sh -c '. /venv/bin/activate && python manage.py migrate --noinput'
-    volumes:
-      - ${PORTAINER_GIT_DIR}:/code
-      - ${NEPTUNE_LOCAL_SETTINGS}:/code/app/local_settings.py
-      - ${NEPTUNE_DATABASE_FILE}:/code/db.sqlite3
-    volumes_from:
-      - tmp
-    depends_on:
-      redis:
-        condition: service_healthy
-
-  tmp:
-    image: busybox
-    command: chmod -R 777 /var/run/redis
-    volumes:
-      - /var/run/redis
-
-  redis:
-    image: redis
-    command: redis-server /etc/redis.conf
-    volumes:
-        - ${PORTAINER_GIT_DIR}/docker/redis.conf:/etc/redis.conf
-    volumes_from:
-        - tmp
-    healthcheck:
-      test: "redis-cli -s /var/run/redis/redis.sock ping"
-      interval: 2s
-      timeout: 2s
-      retries: 15
-
-networks:
-   default:
-      external:
-         name: pathogen

docker/nginx/conf.d/dev.conf

@@ -0,0 +1,23 @@
+upstream django {
+    #server app:8000;
+    #server unix:///var/run/socks/app.sock;
+    server app:8000;
+}
+
+server {
+  listen 9999;
+
+  location = /favicon.ico { access_log off; log_not_found off; }
+
+  location /static/ {
+    root /conf;
+  }
+
+  location / {
+    proxy_pass  http://django;
+    proxy_set_header   X-Real-IP $remote_addr;
+    proxy_set_header   X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header   Host $host;
+  }
+
+}

docker/nginx/conf.d/uwsgi.conf

@@ -0,0 +1,24 @@
+upstream django {
+    server app:8000;
+    #server unix:///var/run/socks/app.sock;
+}
+
+server {
+  listen 9999;
+
+  location = /favicon.ico { access_log off; log_not_found off; }
+
+  location /static/ {
+    root /conf;
+  }
+
+  location / {
+      include     /etc/nginx/uwsgi_params; # the uwsgi_params file you installed
+      uwsgi_pass  django;
+      uwsgi_param Host $host;
+      uwsgi_param X-Real-IP $remote_addr;
+      uwsgi_param X-Forwarded-For $proxy_add_x_forwarded_for;
+      uwsgi_param X-Forwarded-Proto $http_x_forwarded_proto;
+  }
+
+}

docker/prod/Dockerfile

@@ -1,21 +0,0 @@
-# syntax=docker/dockerfile:1
-FROM python:3
-
-RUN useradd -d /code pathogen
-RUN mkdir /code
-RUN chown pathogen:pathogen /code
-
-RUN mkdir /conf
-RUN chown pathogen:pathogen /conf
-
-RUN mkdir /venv
-RUN chown pathogen:pathogen /venv
-
-USER pathogen
-ENV PYTHONDONTWRITEBYTECODE=1
-ENV PYTHONUNBUFFERED=1
-WORKDIR /code
-COPY requirements.prod.txt /code/
-RUN python -m venv /venv
-RUN . /venv/bin/activate && pip install -r requirements.prod.txt
-CMD . /venv/bin/activate && uwsgi --ini /conf/uwsgi.ini

docker/prod/requirements.prod.txt

@@ -1,19 +0,0 @@
-wheel
-django
-django-crispy-forms
-crispy-bulma
-#opensearch-py
-stripe
-django-rest-framework
-numpy
-uwsgi
-django-tables2
-django-tables2-bulma-template
-django-htmx
-cryptography
-siphashc
-redis
-sortedcontainers
-django-debug-toolbar
-django-debug-toolbar-template-profiler
-orjson

docker/redis.conf

@@ -1,2 +1,2 @@
-unixsocket /var/run/redis/redis.sock
+unixsocket /var/run/socks/redis.sock
 unixsocketperm 777

docker/requirements.dev.txt

@@ -1,18 +0,0 @@
-wheel
-django
-django-crispy-forms
-crispy-bulma
-#opensearch-py
-stripe
-django-rest-framework
-numpy
-django-tables2
-django-tables2-bulma-template
-django-htmx
-cryptography
-siphashc
-redis
-sortedcontainers
-django-debug-toolbar
-django-debug-toolbar-template-profiler
-orjson

docker/uwsgi.ini

@@ -5,9 +5,8 @@ env=DJANGO_SETTINGS_MODULE=app.settings
 master=1
 pidfile=/tmp/project-master.pid
 socket=0.0.0.0:8000
-processes=5
 harakiri=20
-max-requests=5000
+max-requests=100000
 vacuum=1
 home=/venv
-
+processes=12

requirements.txt

@@ -1,9 +1,10 @@
 wheel
+uwsgi
 django
 pre-commit
 django-crispy-forms
 crispy-bulma
-#opensearch-py
+elasticsearch
 stripe
 django-rest-framework
 numpy

stack.env

@@ -1,4 +1,6 @@
-NEPTUNE_PORT=5000
+APP_PORT=5000
-PORTAINER_GIT_DIR=..
+PORTAINER_GIT_DIR=.
-NEPTUNE_LOCAL_SETTINGS=../app/local_settings.py
+APP_LOCAL_SETTINGS=./app/local_settings.py
-NEPTUNE_DATABASE_FILE=../db.sqlite3
+APP_DATABASE_FILE=./db.sqlite3
+STATIC_ROOT=/conf/static
+OPERATION=uwsgi