Rename elastic and update settings file

.pre-commit-config.yaml

@@ -24,8 +24,7 @@ repos:
         exclude : ^core/static/css # slow
       - id: djjs
         exclude: ^core/static/js # slow
-#  - repo: https://github.com/thibaudcolas/curlylint
-#    rev: v0.13.1
-#    hooks:
-#      - id: curlylint
-#        files: \.(html|sls)$
+  - repo: https://github.com/sirwart/ripsecrets.git
+    rev: v0.1.5
+    hooks:
+      - id: ripsecrets

app/local_settings.example.py

@@ -1,40 +1,37 @@
 # OpenSearch settings
-OPENSEARCH_URL = "127.0.0.1"
+OPENSEARCH_URL = "10.1.0.1"
 OPENSEARCH_PORT = 9200
 OPENSEARCH_TLS = True
 OPENSEARCH_USERNAME = "admin"
-OPENSEARCH_PASSWORD = ""
-
-OPENSEARCH_INDEX_MAIN = "pathogen-main"
-OPENSEARCH_INDEX_META = "pathogen-meta"
-OPENSEARCH_INDEX_INT = "pathogen-int"
-
-OPENSEARCH_MAIN_SIZES = ["20", "50", "100", "200", "400", "800"]
-OPENSEARCH_MAIN_SIZES_ANON = ["20", "50", "100"]
-OPENSEARCH_MAIN_SOURCES = ["dis", "4ch", "all"]
-OPENSEARCH_SOURCES_RESTRICTED = ["irc"]
+OPENSEARCH_PASSWORD = "secret"
 
 # Manticore settings
-MANTICORE_URL = "http://monolith-db-1:9308"
-MANTICORE_INDEX_MAIN = "main"
-MANTICORE_INDEX_META = "meta"
-MANTICORE_INDEX_INT = "internal"
+MANTICORE_URL = "http://example-db-1:9308"
 
-MANTICORE_MAIN_SIZES = ["20", "50", "100", "200", "400", "800"]
-MANTICORE_MAIN_SIZES_ANON = ["20", "50", "100"]
-MANTICORE_MAIN_SOURCES = ["dis", "4ch", "all"]
-MANTICORE_SOURCES_RESTRICTED = ["irc"]
-MANTICORE_CACHE = True
-MANTICORE_CACHE_TIMEOUT = 60
+DB_BACKEND = "ELASTICSEARCH"
+
+# Common DB settings
+INDEX_MAIN = "main"
+INDEX_RESTRICTED = "restricted"
+INDEX_META = "meta"
+INDEX_INT = "internal"
+
+MAIN_SIZES = ["1", "5", "15", "30", "50", "100", "250", "500", "1000"]
+MAIN_SIZES_ANON = ["1", "5", "15", "30", "50", "100"]
+MAIN_SOURCES = ["dis", "4ch", "all"]
+SOURCES_RESTRICTED = ["irc"]
+CACHE = False
+CACHE_TIMEOUT = 2
 
 DRILLDOWN_RESULTS_PER_PAGE = 15
 DRILLDOWN_DEFAULT_PARAMS = {
-    "size": "20",
+    "size": "15",
     "index": "main",
     "sorting": "desc",
     "source": "4ch",
 }
 
+
 # Encryption
 # ENCRYPTION = False
 # ENCRYPTION_KEY = b""
@@ -61,7 +58,8 @@ DRILLDOWN_DEFAULT_PARAMS = {
 # # Delay results by this many days
 # DELAY_DURATION = 10
 
-OPENSEARCH_BLACKLISTED = {}
+OPENSEARCH_BLACKLISTED = {
+                         }
 
 
 # URLs\
@@ -89,8 +87,8 @@ SECRET_KEY = "a"
 STRIPE_ADMIN_COUPON = ""
 
 # Threshold
-THRESHOLD_ENDPOINT = "http://threshold-app-1:13869"
-THRESHOLD_API_KEY = ""
+THRESHOLD_ENDPOINT = "http://threshold:13869"
+THRESHOLD_API_KEY = "api_1"
 THRESHOLD_API_TOKEN = ""
 THRESHOLD_API_COUNTER = ""
 
@@ -105,13 +103,4 @@ META_MAX_CHUNK_SIZE = 500
 META_QUERY_SIZE = 10000
 
 DEBUG = True
-PROFILER = False
-
-if DEBUG:
-    import socket  # only if you haven't already imported this
-
-    hostname, _, ips = socket.gethostbyname_ex(socket.gethostname())
-    INTERNAL_IPS = [ip[: ip.rfind(".")] + ".1" for ip in ips] + [
-        "127.0.0.1",
-        "10.0.2.2",
-    ]
+PROFILER = False

core/db/druid.py

@@ -114,7 +114,7 @@ class DruidBackend(StorageBackend):
         ss = orjson.dumps(search_query, option=orjson.OPT_INDENT_2)
         ss = ss.decode()
         print(ss)
-        response = requests.post("http://broker:8082/druid/v2", json=search_query)
+        response = requests.post("http://druid:8082/druid/v2", json=search_query)
         response = orjson.loads(response.text)
         print("RESPONSE LEN", len(response))
         # ss = orjson.dumps(response, option=orjson.OPT_INDENT_2)

core/db/elastic.py

@@ -13,9 +13,9 @@ from core.db.processing import annotate_results, parse_results
 from core.views.helpers import dedup_list
 
 
-class OpensearchBackend(StorageBackend):
+class ElasticsearchBackend(StorageBackend):
     def __init__(self):
-        super().__init__("Opensearch")
+        super().__init__("Elasticsearch")
 
     def initialise(self, **kwargs):
         """

core/db/storage.py

@@ -6,8 +6,8 @@ def get_db():
         from core.db.druid import DruidBackend
 
         return DruidBackend()
-    elif settings.DB_BACKEND == "OPENSEARCH":
-        from core.db.opensearch import OpensearchBackend
+    elif settings.DB_BACKEND == "ELASTICSEARCH":
+        from core.db.elastic import ElasticSearchBackend
 
         return OpensearchBackend()
     elif settings.DB_BACKEND == "MANTICORE":

core/static/js/column-shifter.js

@@ -65,11 +65,12 @@ $(document).ready(function(){
                     "file_ext": "off",
                     "file_size": "off",
                     "lang_code": "off",
+                    "tokens": "off",
                     //"lang_name": "off",
-                    "words_noun": "off",
-                    "words_adj": "off",
-                    "words_verb": "off",
-                    "words_adv": "off"
+                    // "words_noun": "off",
+                    // "words_adj": "off",
+                    // "words_verb": "off",
+                    // "words_adv": "off"
                 },
             };
         } else {

core/views/ui/tables.py

@@ -64,14 +64,13 @@ class DrilldownTable(Table):
     mtype = Column()
     realname = Column()
     server = Column()
-    mtype = Column()
-    # tokens = Column()
+    tokens = Column()
     lang_code = Column()
     lang_name = Column()
-    words_noun = Column()
-    words_adj = Column()
-    words_verb = Column()
-    words_adv = Column()
+    # words_noun = Column()
+    # words_adj = Column()
+    # words_verb = Column()
+    # words_adv = Column()
     hidden = Column()
     filename = Column()
     file_md5 = Column()