XF/monolith
2
0
Fork 0
Code Issues 2 Pull Requests Projects Releases Wiki Activity

Properly process Redis buffered messages and ingest into Kafka

Browse Source
This commit is contained in:
Mark Veidemanis
2022-09-14 18:32:32 +01:00
parent fec0d379a6
commit 4ea77ac543
6 changed files with 190 additions and 133 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
sources/ingest.py
View File

@@ -4,24 +4,22 @@ import ujson
import db
import util
from processing import process
SOURCES = ["irc", "dis", "4ch"]
SOURCES = ["4ch", "irc", "dis"]
KEYPREFIX = "queue."
CHUNK_SIZE = 1000
CHUNK_SIZE = 90000
ITER_DELAY = 0.5
class Ingest(object):
def __init__(self):
name = self.__class__.__name__
self.log = util.get_logger(name)
async def run(self):
# items = [{'no': 23567753, 'now': '09/12/22(Mon)20:10:29', 'name': 'Anonysmous', 'filename': '1644986767568', 'ext': '.webm', 'w': 1280, 'h': 720, 'tn_w': 125, 'tn_h': 70, 'tim': 1663027829301457, 'time': 1663027829, 'md5': 'zeElr1VR05XpZ2XuAPhmPA==', 'fsize': 3843621, 'resto': 23554700, 'type': 'msg', 'src': '4ch', 'net': 'gif', 'channel': '23554700'}]
# await process.spawn_processing_threads(items)
while True:
await self.get_chunk()
await asyncio.sleep(ITER_DELAY)
@@ -33,13 +31,11 @@ class Ingest(object):
chunk = await db.ar.spop(key, CHUNK_SIZE)
if not chunk:
continue
#self.log.info(f"Got chunk: {chunk}")
# self.log.info(f"Got chunk: {chunk}")
for item in chunk:
item = ujson.loads(item)
#self.log.info(f"Got item: {item}")
# self.log.info(f"Got item: {item}")
items.append(item)
if items:
print("PROCESSING", len(items))
await process.spawn_processing_threads(items)
print("DONE WITH PROCESSING", len(items))
await db.store_kafka_batch(items)