XF/monolith
2
0
Fork 0
Code Issues 2 Pull Requests Projects Releases Wiki Activity

Properly process Redis buffered messages and ingest into Kafka

Browse Source
This commit is contained in:
Mark Veidemanis
2022-09-14 18:32:32 +01:00
parent fec0d379a6
commit 4ea77ac543
6 changed files with 190 additions and 133 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
sources/ch4.py
View File

@@ -136,7 +136,7 @@ class Chan4(object):
# Split into 10,000 chunks
if not all_posts:
return
self.handle_posts(all_posts)
await self.handle_posts(all_posts)
# threads_per_core = int(len(all_posts) / CPU_THREADS)
# for i in range(CPU_THREADS):
# new_dict = {}
@@ -146,8 +146,7 @@ class Chan4(object):
# new_dict[k].append(v)
# else:
# new_dict[k] = [v]
#await self.handle_posts_thread(new_dict)
# await self.handle_posts_thread(new_dict)
# print("VAL", ceil(len(all_posts) / threads_per_core))
# split_posts = array_split(all_posts, ceil(len(all_posts) / threads_per_core))

16
sources/ingest.py
View File

@@ -4,24 +4,22 @@ import ujson
import db
import util
from processing import process
SOURCES = ["irc", "dis", "4ch"]
SOURCES = ["4ch", "irc", "dis"]
KEYPREFIX = "queue."
CHUNK_SIZE = 1000
CHUNK_SIZE = 90000
ITER_DELAY = 0.5
class Ingest(object):
def __init__(self):
name = self.__class__.__name__
self.log = util.get_logger(name)
async def run(self):
# items = [{'no': 23567753, 'now': '09/12/22(Mon)20:10:29', 'name': 'Anonysmous', 'filename': '1644986767568', 'ext': '.webm', 'w': 1280, 'h': 720, 'tn_w': 125, 'tn_h': 70, 'tim': 1663027829301457, 'time': 1663027829, 'md5': 'zeElr1VR05XpZ2XuAPhmPA==', 'fsize': 3843621, 'resto': 23554700, 'type': 'msg', 'src': '4ch', 'net': 'gif', 'channel': '23554700'}]
# await process.spawn_processing_threads(items)
while True:
await self.get_chunk()
await asyncio.sleep(ITER_DELAY)
@@ -33,13 +31,11 @@ class Ingest(object):
chunk = await db.ar.spop(key, CHUNK_SIZE)
if not chunk:
continue
#self.log.info(f"Got chunk: {chunk}")
# self.log.info(f"Got chunk: {chunk}")
for item in chunk:
item = ujson.loads(item)
#self.log.info(f"Got item: {item}")
# self.log.info(f"Got item: {item}")
items.append(item)
if items:
print("PROCESSING", len(items))
await process.spawn_processing_threads(items)
print("DONE WITH PROCESSING", len(items))
await db.store_kafka_batch(items)