Pathogen
/
monolith
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Remove commented debug code

This commit is contained in:
Mark Veidemanis 2022-09-30 07:22:22 +01:00
parent 02ff44a6f5
commit 5c91f1af87
2 changed files with 1 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
db.py
View File

@ -113,11 +113,7 @@ async def queue_message(msg):
""" """
Queue a message on the Redis buffer. Queue a message on the Redis buffer.
""" """
# src = msg["src"]
message = orjson.dumps(msg) message = orjson.dumps(msg)
# key = f"{KEYPREFIX}{src}"
# log.debug(f"Queueing single message of string length {len(message)}")
await ar.sadd(KEYNAME, message) await ar.sadd(KEYNAME, message)
@ -125,10 +121,6 @@ async def queue_message_bulk(data):
""" """
Queue multiple messages on the Redis buffer. Queue multiple messages on the Redis buffer.
""" """
# log.debug(f"Queueing message batch of length {len(data)}")
for msg in data: for msg in data:
# src = msg["src"]
message = orjson.dumps(msg) message = orjson.dumps(msg)
# key = f"{KEYPREFIX}{src}"
await ar.sadd(KEYNAME, message) await ar.sadd(KEYNAME, message)

8
sources/ingest.py
View File

@ -7,15 +7,9 @@ import db
import util import util
from processing import process from processing import process
# SOURCES = ["4ch", "irc", "dis"]
# DEBUG CODE REMOVE ME
# SOURCES.remove("4ch")
# SOURCES.remove("dis")
# DEBUG CODE REMOVE ME
# KEYPREFIX = "queue."
KEYNAME = "queue" KEYNAME = "queue"
# Chunk size per source (divide by len(SOURCES) for total) # Chunk size
CHUNK_SIZE = int(getenv("MONOLITH_INGEST_CHUNK_SIZE", "900")) CHUNK_SIZE = int(getenv("MONOLITH_INGEST_CHUNK_SIZE", "900"))
ITER_DELAY = float(getenv("MONOLITH_INGEST_ITER_DELAY", "0.5")) ITER_DELAY = float(getenv("MONOLITH_INGEST_ITER_DELAY", "0.5"))