Ingest into Kafka and queue messages better
This commit is contained in:
102
sources/ch4.py
102
sources/ch4.py
@@ -19,19 +19,19 @@ from schemas.ch4_s import ATTRMAP
|
||||
# CONFIGURATION #
|
||||
|
||||
# Number of 4chan threads to request at once
|
||||
THREADS_CONCURRENT = 100
|
||||
THREADS_CONCURRENT = 1000
|
||||
|
||||
# Seconds to wait between every THREADS_CONCURRENT requests
|
||||
THREADS_DELAY = 0.8
|
||||
THREADS_DELAY = 0.1
|
||||
|
||||
# Seconds to wait between crawls
|
||||
CRAWL_DELAY = 5
|
||||
|
||||
# Semaphore value ?
|
||||
THREADS_SEMAPHORE = 100
|
||||
THREADS_SEMAPHORE = 1000
|
||||
|
||||
# Maximum number of CPU threads to use for post processing
|
||||
CPU_THREADS = 1
|
||||
CPU_THREADS = 8
|
||||
|
||||
# CONFIGURATION END #
|
||||
|
||||
@@ -95,7 +95,7 @@ class Chan4(object):
|
||||
no = threads["no"]
|
||||
to_get.append((mapped, no))
|
||||
|
||||
self.log.info(f"Got thread list for {mapped}: {len(response)}")
|
||||
self.log.debug(f"Got thread list for {mapped}: {len(response)}")
|
||||
if not to_get:
|
||||
return
|
||||
split_threads = array_split(to_get, ceil(len(to_get) / THREADS_CONCURRENT))
|
||||
@@ -136,16 +136,19 @@ class Chan4(object):
|
||||
# Split into 10,000 chunks
|
||||
if not all_posts:
|
||||
return
|
||||
threads_per_core = int(len(all_posts) / CPU_THREADS)
|
||||
for i in range(CPU_THREADS):
|
||||
new_dict = {}
|
||||
pulled_posts = self.take_items(all_posts, threads_per_core)
|
||||
for k, v in pulled_posts:
|
||||
if k in new_dict:
|
||||
new_dict[k].append(v)
|
||||
else:
|
||||
new_dict[k] = [v]
|
||||
await self.handle_posts_thread(new_dict)
|
||||
self.handle_posts(all_posts)
|
||||
# threads_per_core = int(len(all_posts) / CPU_THREADS)
|
||||
# for i in range(CPU_THREADS):
|
||||
# new_dict = {}
|
||||
# pulled_posts = self.take_items(all_posts, threads_per_core)
|
||||
# for k, v in pulled_posts:
|
||||
# if k in new_dict:
|
||||
# new_dict[k].append(v)
|
||||
# else:
|
||||
# new_dict[k] = [v]
|
||||
#await self.handle_posts_thread(new_dict)
|
||||
|
||||
|
||||
# print("VAL", ceil(len(all_posts) / threads_per_core))
|
||||
# split_posts = array_split(all_posts, ceil(len(all_posts) / threads_per_core))
|
||||
# print("THREADS PER CORE SPLIT", len(split_posts))
|
||||
@@ -161,46 +164,46 @@ class Chan4(object):
|
||||
loop = asyncio.get_event_loop()
|
||||
yield from loop.run_in_executor(p, self.handle_posts, posts)
|
||||
|
||||
def handle_posts(self, posts):
|
||||
async def handle_posts(self, posts):
|
||||
to_store = []
|
||||
for key, post_list in posts.items():
|
||||
board, thread = key
|
||||
for index, post in enumerate(post_list):
|
||||
posts[key][index]["type"] = "msg"
|
||||
|
||||
# Calculate hash for post
|
||||
post_normalised = ujson.dumps(post, sort_keys=True)
|
||||
hash = siphash(self.hash_key, post_normalised)
|
||||
hash = str(hash)
|
||||
redis_key = f"cache.{board}.{thread}.{post['no']}"
|
||||
key_content = db.r.get(redis_key)
|
||||
if key_content:
|
||||
key_content = key_content.decode("ascii")
|
||||
if key_content == hash:
|
||||
continue
|
||||
else:
|
||||
posts[key][index]["type"] = "update"
|
||||
db.r.set(redis_key, hash)
|
||||
# # Calculate hash for post
|
||||
# post_normalised = ujson.dumps(post, sort_keys=True)
|
||||
# hash = siphash(self.hash_key, post_normalised)
|
||||
# hash = str(hash)
|
||||
# redis_key = f"cache.{board}.{thread}.{post['no']}"
|
||||
# key_content = db.r.get(redis_key)
|
||||
# if key_content:
|
||||
# key_content = key_content.decode("ascii")
|
||||
# if key_content == hash:
|
||||
# continue
|
||||
# else:
|
||||
# posts[key][index]["type"] = "update"
|
||||
# #db.r.set(redis_key, hash)
|
||||
|
||||
for key2, value in list(post.items()):
|
||||
if key2 in ATTRMAP:
|
||||
post[ATTRMAP[key2]] = posts[key][index][key2]
|
||||
del posts[key][index][key2]
|
||||
if "ts" in post:
|
||||
old_time = posts[key][index]["ts"]
|
||||
# '08/30/22(Tue)02:25:37'
|
||||
time_spl = old_time.split(":")
|
||||
if len(time_spl) == 3:
|
||||
old_ts = datetime.strptime(old_time, "%m/%d/%y(%a)%H:%M:%S")
|
||||
else:
|
||||
old_ts = datetime.strptime(old_time, "%m/%d/%y(%a)%H:%M")
|
||||
# new_ts = old_ts.isoformat()
|
||||
new_ts = int(old_ts.timestamp())
|
||||
posts[key][index]["ts"] = new_ts
|
||||
if "msg" in post:
|
||||
soup = BeautifulSoup(posts[key][index]["msg"], "html.parser")
|
||||
msg = soup.get_text(separator="\n")
|
||||
posts[key][index]["msg"] = msg
|
||||
# for key2, value in list(post.items()):
|
||||
# if key2 in ATTRMAP:
|
||||
# post[ATTRMAP[key2]] = posts[key][index][key2]
|
||||
# del posts[key][index][key2]
|
||||
# if "ts" in post:
|
||||
# old_time = posts[key][index]["ts"]
|
||||
# # '08/30/22(Tue)02:25:37'
|
||||
# time_spl = old_time.split(":")
|
||||
# if len(time_spl) == 3:
|
||||
# old_ts = datetime.strptime(old_time, "%m/%d/%y(%a)%H:%M:%S")
|
||||
# else:
|
||||
# old_ts = datetime.strptime(old_time, "%m/%d/%y(%a)%H:%M")
|
||||
# # new_ts = old_ts.isoformat()
|
||||
# new_ts = int(old_ts.timestamp())
|
||||
# posts[key][index]["ts"] = new_ts
|
||||
# if "msg" in post:
|
||||
# soup = BeautifulSoup(posts[key][index]["msg"], "html.parser")
|
||||
# msg = soup.get_text(separator="\n")
|
||||
# posts[key][index]["msg"] = msg
|
||||
|
||||
posts[key][index]["src"] = "4ch"
|
||||
posts[key][index]["net"] = board
|
||||
@@ -211,7 +214,8 @@ class Chan4(object):
|
||||
# print({name_map[name]: val for name, val in post.items()})
|
||||
# print(f"Got posts: {len(posts)}")
|
||||
if to_store:
|
||||
db.store_message_bulk(to_store)
|
||||
print("STORING", len(to_store))
|
||||
await db.queue_message_bulk(to_store)
|
||||
|
||||
async def fetch(self, url, session, mapped):
|
||||
async with session.get(url) as response:
|
||||
|
||||
Reference in New Issue
Block a user