|
|
|
@ -1,27 +1,27 @@
|
|
|
|
|
from concurrent.futures import ProcessPoolExecutor
|
|
|
|
|
import asyncio
|
|
|
|
|
import os
|
|
|
|
|
import ujson
|
|
|
|
|
from siphashc import siphash
|
|
|
|
|
|
|
|
|
|
import db
|
|
|
|
|
import util
|
|
|
|
|
|
|
|
|
|
# 4chan schema
|
|
|
|
|
from schemas.ch4_s import ATTRMAP
|
|
|
|
|
import random
|
|
|
|
|
|
|
|
|
|
# For key generation
|
|
|
|
|
import string
|
|
|
|
|
import random
|
|
|
|
|
from concurrent.futures import ProcessPoolExecutor
|
|
|
|
|
|
|
|
|
|
# For timestamp processing
|
|
|
|
|
import datetime
|
|
|
|
|
from datetime import datetime
|
|
|
|
|
from math import ceil
|
|
|
|
|
|
|
|
|
|
import ujson
|
|
|
|
|
|
|
|
|
|
# For 4chan message parsing
|
|
|
|
|
from bs4 import BeautifulSoup
|
|
|
|
|
|
|
|
|
|
from numpy import array_split
|
|
|
|
|
from math import ceil
|
|
|
|
|
from siphashc import siphash
|
|
|
|
|
|
|
|
|
|
import db
|
|
|
|
|
import util
|
|
|
|
|
|
|
|
|
|
# 4chan schema
|
|
|
|
|
from schemas.ch4_s import ATTRMAP
|
|
|
|
|
|
|
|
|
|
log = util.get_logger("process")
|
|
|
|
|
|
|
|
|
@ -30,6 +30,7 @@ CPU_THREADS = os.cpu_count()
|
|
|
|
|
|
|
|
|
|
p = ProcessPoolExecutor(CPU_THREADS)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def get_hash_key():
|
|
|
|
|
hash_key = db.r.get("hashing_key")
|
|
|
|
|
if not hash_key:
|
|
|
|
@ -42,31 +43,66 @@ def get_hash_key():
|
|
|
|
|
log.debug(f"Decoded hash key: {hash_key}")
|
|
|
|
|
return hash_key
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
hash_key = get_hash_key()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@asyncio.coroutine
|
|
|
|
|
async def spawn_processing_threads(data):
|
|
|
|
|
print("SPAWN", data)
|
|
|
|
|
loop = asyncio.get_event_loop()
|
|
|
|
|
tasks = []
|
|
|
|
|
oldts = [x["now"] for x in data if "now" in x]
|
|
|
|
|
if len(data) < CPU_THREADS:
|
|
|
|
|
split_data = [data]
|
|
|
|
|
else:
|
|
|
|
|
msg_per_core = int(len(data) / CPU_THREADS)
|
|
|
|
|
print("MSG PER CORE", msg_per_core)
|
|
|
|
|
split_data = array_split(data, ceil(len(data) / msg_per_core))
|
|
|
|
|
print("SPLIT DATA", split_data)
|
|
|
|
|
for split in split_data:
|
|
|
|
|
for index, split in enumerate(split_data):
|
|
|
|
|
print("DELEGATING TO THREAD", len(split))
|
|
|
|
|
await process_data_thread(split)
|
|
|
|
|
future = loop.run_in_executor(p, process_data, data)
|
|
|
|
|
# future = p.submit(process_data, split)
|
|
|
|
|
tasks.append(future)
|
|
|
|
|
# results = [x.result(timeout=50) for x in tasks]
|
|
|
|
|
results = await asyncio.gather(*tasks)
|
|
|
|
|
print("RESULTS", len(results))
|
|
|
|
|
|
|
|
|
|
# Join the results back from the split list
|
|
|
|
|
flat_list = [item for sublist in results for item in sublist]
|
|
|
|
|
print("LENFLAT", len(flat_list))
|
|
|
|
|
print("LENDATA", len(data))
|
|
|
|
|
|
|
|
|
|
newts = [x["ts"] for x in flat_list if "ts" in x]
|
|
|
|
|
print("lenoldts", len(oldts))
|
|
|
|
|
print("lennewts", len(newts))
|
|
|
|
|
allts = all(["ts" in x for x in flat_list])
|
|
|
|
|
print("ALLTS", allts)
|
|
|
|
|
alllen = [len(x) for x in flat_list]
|
|
|
|
|
print("ALLLEN", alllen)
|
|
|
|
|
await db.store_kafka_batch(flat_list)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# @asyncio.coroutine
|
|
|
|
|
# def process_data_thread(data):
|
|
|
|
|
# """
|
|
|
|
|
# Helper to spawn threads to process a list of data.
|
|
|
|
|
# """
|
|
|
|
|
# loop = asyncio.get_event_loop()
|
|
|
|
|
# if len(data) < CPU_THREADS:
|
|
|
|
|
# split_data = [data]
|
|
|
|
|
# else:
|
|
|
|
|
# msg_per_core = int(len(data) / CPU_THREADS)
|
|
|
|
|
# print("MSG PER CORE", msg_per_core)
|
|
|
|
|
# split_data = array_split(data, ceil(len(data) / msg_per_core))
|
|
|
|
|
# for index, split in enumerate(split_data):
|
|
|
|
|
# print("DELEGATING TO THREAD", len(split))
|
|
|
|
|
# #f = process_data_thread(split)
|
|
|
|
|
# yield loop.run_in_executor(p, process_data, data)
|
|
|
|
|
|
|
|
|
|
@asyncio.coroutine
|
|
|
|
|
def process_data_thread(data):
|
|
|
|
|
"""
|
|
|
|
|
Helper to spawn threads to process a list of data.
|
|
|
|
|
"""
|
|
|
|
|
loop = asyncio.get_event_loop()
|
|
|
|
|
yield from loop.run_in_executor(p, process_data, data)
|
|
|
|
|
|
|
|
|
|
def process_data(data):
|
|
|
|
|
print("PROCESSING DATA", data)
|
|
|
|
|
print("PROCESS DATA START")
|
|
|
|
|
# to_store = []
|
|
|
|
|
for index, msg in enumerate(data):
|
|
|
|
|
# print("PROCESSING", msg)
|
|
|
|
|
if msg["src"] == "4ch":
|
|
|
|
@ -81,15 +117,18 @@ def process_data(data):
|
|
|
|
|
if key_content:
|
|
|
|
|
key_content = key_content.decode("ascii")
|
|
|
|
|
if key_content == hash:
|
|
|
|
|
del data[index]
|
|
|
|
|
continue
|
|
|
|
|
else:
|
|
|
|
|
data[index][index]["type"] = "update"
|
|
|
|
|
data[index]["type"] = "update"
|
|
|
|
|
db.r.set(redis_key, hash)
|
|
|
|
|
for key2, value in list(msg.items()):
|
|
|
|
|
if "now" not in data[index]:
|
|
|
|
|
print("NOW NOT IN INDEX", data[index])
|
|
|
|
|
for key2, value in list(data[index].items()):
|
|
|
|
|
if key2 in ATTRMAP:
|
|
|
|
|
msg[ATTRMAP[key2]] = data[index][key2]
|
|
|
|
|
data[index][ATTRMAP[key2]] = data[index][key2]
|
|
|
|
|
del data[index][key2]
|
|
|
|
|
if "ts" in msg:
|
|
|
|
|
if "ts" in data[index]:
|
|
|
|
|
old_time = data[index]["ts"]
|
|
|
|
|
# '08/30/22(Tue)02:25:37'
|
|
|
|
|
time_spl = old_time.split(":")
|
|
|
|
@ -100,7 +139,13 @@ def process_data(data):
|
|
|
|
|
# new_ts = old_ts.isoformat()
|
|
|
|
|
new_ts = int(old_ts.timestamp())
|
|
|
|
|
data[index]["ts"] = new_ts
|
|
|
|
|
else:
|
|
|
|
|
print("MSG WITHOUT TS PROCESS", data[index])
|
|
|
|
|
continue
|
|
|
|
|
if "msg" in msg:
|
|
|
|
|
soup = BeautifulSoup(data[index]["msg"], "html.parser")
|
|
|
|
|
msg = soup.get_text(separator="\n")
|
|
|
|
|
data[index]["msg"] = msg
|
|
|
|
|
# to_store.append(data[index])
|
|
|
|
|
print("FINISHED PROCESSING DATA")
|
|
|
|
|
return data
|
|
|
|
|