Port Manticore and debug

.gitignore

@@ -157,3 +157,4 @@ cython_debug/
 .vscode/
 stack.env
 
+static/

app/settings.py

@@ -179,11 +179,12 @@ CACHES = {
     "default": {
         "BACKEND": "django_redis.cache.RedisCache",
         # "LOCATION": "unix:///var/run/socks/redis.sock",
-        "LOCATION": f"redis://{REDIS_HOST}:{REDIS_PORT}",
+        # "LOCATION": f"redis://{REDIS_HOST}:{REDIS_PORT}",
+        "LOCATION": "unix:///var/run/neptune-redis.sock",
         "OPTIONS": {
-            "db": REDIS_DB_CACHE,
+            "db": REDIS_DB,
             # "parser_class": "django_redis.cache.RedisCache",
-            "PASSWORD": REDIS_PASSWORD,
+            # "PASSWORD": REDIS_PASSWORD,
             "pool_class": "redis.BlockingConnectionPool",
         },
     }

app/urls.py

@@ -95,7 +95,7 @@ urlpatterns = [
     ),
     path("cancel/", TemplateView.as_view(template_name="cancel.html"), name="cancel"),
     path("portal", Portal.as_view(), name="portal"),
-    path("admin/", admin.site.urls),
+    path("sapp/", admin.site.urls),
     path("accounts/", include("django.contrib.auth.urls")),
     path("accounts/signup/", Signup.as_view(), name="signup"),
     ##

core/__init__.py

@@ -6,13 +6,15 @@ from redis import StrictRedis
 
 os.environ["DJANGO_ALLOW_ASYNC_UNSAFE"] = "true"
 
-
+# /var/run/neptune-redis.sock
-r = StrictRedis(
+# use the socket
-    host=settings.REDIS_HOST,
+r = StrictRedis(unix_socket_path="/var/run/neptune-redis.sock", db=settings.REDIS_DB)
-    port=settings.REDIS_PORT,
+# r = StrictRedis(
-    password=settings.REDIS_PASSWORD,
+#     host=settings.REDIS_HOST,
-    db=settings.REDIS_DB
+#     port=settings.REDIS_PORT,
-)
+#     password=settings.REDIS_PASSWORD,
+#     db=settings.REDIS_DB
+# )
 
 if settings.STRIPE_TEST:
     stripe.api_key = settings.STRIPE_API_KEY_TEST

core/db/__init__.py

@@ -253,7 +253,6 @@ class StorageBackend(ABC):
                     "took": time_took_rounded,
                     "cache": True,
                 }
-        print("S2", search_query)
         response = self.run_query(user, search_query, **kwargs)
 
         # For Elasticsearch

core/db/manticore.py

@@ -57,7 +57,6 @@ class ManticoreBackend(StorageBackend):
             "limit": size,
             "query": {"bool": {"must": []}},
         }
-        print("BASE", query_base)
         query_string = {
             "query_string": query,
         }
@@ -77,7 +76,6 @@ class ManticoreBackend(StorageBackend):
         raw = kwargs.get("raw")
         if search_query and not raw:
             search_query["index"] = index
-            pprint(search_query)
 
 
         path = kwargs.get("path", "json/search")
@@ -98,7 +96,6 @@ class ManticoreBackend(StorageBackend):
         """
         index = kwargs.get("index")
         search_query["index"] = index
-        pprint(search_query)
         async with httpx.AsyncClient() as client:
             response = await client.post(
                 f"{settings.MANTICORE_URL}/json/search", json=search_query

core/db/processing.py

@@ -94,7 +94,7 @@ def parse_results(results, meta=None):
                         del element[field]
                 # Unfold the tokens
                 if "tokens" in element:
-                    if element["tokens"].startswith('["'):
+                    if element["tokens"].startswith('["') or element["tokens"].startswith("['"):
                         tokens_parsed = ast.literal_eval(element["tokens"])
                         element["tokens"] = tokens_parsed
 

core/lib/context.py

@@ -85,5 +85,4 @@ def construct_query(index, net, channel, src, num, size, type=None, nicks=None):
     if extra_should2:
         query["query"]["bool"]["must"].append({"bool": {"should": [*extra_should2]}})
 
-    print("CONTEXT QUERY", query)
     return query

core/lib/parsing.py

@@ -91,7 +91,6 @@ def parse_index(user, query_params, raise_error=False):
     else:
         index = settings.INDEX_MAIN
 
-    print("GOT INDEX", index)
     return index
 
 

core/management/commands/processing.py

@@ -94,12 +94,13 @@ def process_rules(data):
 
 class Command(BaseCommand):
     def handle(self, *args, **options):
-        r = StrictRedis(
+        r = StrictRedis(unix_socket_path="/var/run/neptune-redis.sock", db=10) # To match Monolith DB
-            host=settings.REDIS_HOST,
+        # r = StrictRedis(
-            port=settings.REDIS_PORT,
+        #     host=settings.REDIS_HOST,
-            password=settings.REDIS_PASSWORD,
+        #     port=settings.REDIS_PORT,
-            db=settings.REDIS_DB
+        #     password=settings.REDIS_PASSWORD,
-        )
+        #     db=settings.REDIS_DB
+        # )
         p = r.pubsub()
         p.psubscribe("messages")
         for message in p.listen():

core/management/commands/scheduling.py

@@ -44,8 +44,11 @@ class Command(BaseCommand):
         for interval in INTERVALS:
             log.debug(f"Scheduling {interval} second job")
             scheduler.add_job(job, "interval", seconds=interval, args=[interval])
+
+        loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(loop)
+        scheduler._eventloop = loop
         scheduler.start()
-        loop = asyncio.get_event_loop()
         try:
             loop.run_forever()
         except (KeyboardInterrupt, SystemExit):

core/templates/partials/sentiment_chart.html

@@ -3,7 +3,7 @@
 </div>
 
 {% if params.index != 'int' and params.index != 'meta' %}
-  <div id="sentiment-container" {% if params.show_sentiment is None %} class="is-hidden" {% endif %}>
+  <div id="sentiment-container" {% if params.graph is None %} class="is-hidden" {% endif %}>
     <canvas id="sentiment-chart"></canvas>
   </div>
   <script src="{% static 'chart.js' %}"></script>

core/templates/window-content/search.html

@@ -258,7 +258,7 @@
               id="sentiment_graph_switch"
               type="checkbox"
               class="switch is-rounded is-info"
-              name="show_sentiment"
+              name="graph"
               data-script="on click toggle .is-hidden on #sentiment-container">
 
             <label

core/views/ui/drilldown.py

@@ -119,16 +119,20 @@ class DrilldownTableView(SingleTableView):
             sizes = settings.MAIN_SIZES
 
         if request.GET:
+            print("GET")
             self.template_name = "index.html"
             # GET arguments in URL like ?query=xyz
             query_params = request.GET.dict()
+            print("QUERY_PARAMS GET", query_params)
             if request.htmx:
                 if request.resolver_match.url_name == "search_partial":
                     self.template_name = "partials/results_table.html"
         elif request.POST:
+            print("POST")
             query_params = request.POST.dict()
         else:
             self.template_name = "index.html"
+            print("FRESH")
             # No query, this is a fresh page load
             # Don't try to search, since there's clearly nothing to do
             params_with_defaults = {}

docker/redis.conf

@@ -1,4 +1,5 @@
-# unixsocket /var/run/socks/redis.sock
+unixsocket /var/run/neptune-redis.sock
-# unixsocketperm 777
+unixsocketperm 777
-port 6379
+port 0
-requirepass changeme
+# port 6379
+# requirepass changeme

docker/uwsgi.ini

@@ -4,9 +4,19 @@ module=app.wsgi:application
 env=DJANGO_SETTINGS_MODULE=app.settings
 master=1
 pidfile=/tmp/project-master.pid
-socket=0.0.0.0:8000
+#socket=0.0.0.0:8000
+socket=/var/run/uwsgi-neptune.sock
+# socket 777
+chmod-socket=777
 harakiri=20
-max-requests=100000
+#max-requests=100000
+# Set a lower value for max-requests to prevent memory leaks from building up over time
+max-requests=1000
+# Ensure old worker processes are cleaned up properly
+reload-on-as=512
+reload-on-rss=256
 vacuum=1
 home=/venv
-processes=12
+processes=4
+threads=2
+log-level=debug