Make performance settings configurable

sources/ingest.py

@@ -5,13 +5,14 @@ import orjson
 import db
 import util
 from processing import process
+from os import getenv
 
 SOURCES = ["4ch", "irc", "dis"]
 KEYPREFIX = "queue."
 
 # Chunk size per source (divide by len(SOURCES) for total)
-CHUNK_SIZE = 9000
-ITER_DELAY = 0.5
+CHUNK_SIZE = int(getenv("MONOLITH_INGEST_CHUNK_SIZE", "900"))
+ITER_DELAY = float(getenv("MONOLITH_INGEST_ITER_DELAY", "0.5"))
 
 log = util.get_logger("ingest")