|
|
|
@ -4,25 +4,73 @@ import random
|
|
|
|
|
|
|
|
|
|
# For key generation
|
|
|
|
|
import string
|
|
|
|
|
|
|
|
|
|
# Squash errors
|
|
|
|
|
import warnings
|
|
|
|
|
from concurrent.futures import ProcessPoolExecutor
|
|
|
|
|
|
|
|
|
|
# For timestamp processing
|
|
|
|
|
from datetime import datetime
|
|
|
|
|
from math import ceil
|
|
|
|
|
|
|
|
|
|
import ujson
|
|
|
|
|
import orjson
|
|
|
|
|
|
|
|
|
|
# Tokenisation
|
|
|
|
|
import spacy
|
|
|
|
|
|
|
|
|
|
# For 4chan message parsing
|
|
|
|
|
from bs4 import BeautifulSoup
|
|
|
|
|
from numpy import array_split
|
|
|
|
|
from polyglot.detect.base import logger as polyglot_logger
|
|
|
|
|
|
|
|
|
|
# For NLP
|
|
|
|
|
from polyglot.text import Text
|
|
|
|
|
from pycld2 import error as cld2_error
|
|
|
|
|
from siphashc import siphash
|
|
|
|
|
|
|
|
|
|
# For sentiment
|
|
|
|
|
from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer
|
|
|
|
|
|
|
|
|
|
import db
|
|
|
|
|
import util
|
|
|
|
|
|
|
|
|
|
# 4chan schema
|
|
|
|
|
from schemas.ch4_s import ATTRMAP
|
|
|
|
|
|
|
|
|
|
# For tokenisation
|
|
|
|
|
# from gensim.parsing.preprocessing import (
|
|
|
|
|
# strip_tags,
|
|
|
|
|
# strip_punctuation,
|
|
|
|
|
# strip_numeric,
|
|
|
|
|
# stem_text,
|
|
|
|
|
# strip_multiple_whitespaces,
|
|
|
|
|
# strip_non_alphanum,
|
|
|
|
|
# remove_stopwords,
|
|
|
|
|
# strip_short,
|
|
|
|
|
# preprocess_string,
|
|
|
|
|
# )
|
|
|
|
|
|
|
|
|
|
# CUSTOM_FILTERS = [
|
|
|
|
|
# lambda x: x.lower(),
|
|
|
|
|
# strip_tags, #
|
|
|
|
|
# strip_punctuation, #
|
|
|
|
|
# strip_multiple_whitespaces,
|
|
|
|
|
# strip_numeric,
|
|
|
|
|
# remove_stopwords,
|
|
|
|
|
# strip_short,
|
|
|
|
|
# #stem_text,
|
|
|
|
|
# strip_non_alphanum, #
|
|
|
|
|
# ]
|
|
|
|
|
|
|
|
|
|
# Squash errors
|
|
|
|
|
polyglot_logger.setLevel("ERROR")
|
|
|
|
|
warnings.filterwarnings("ignore", category=UserWarning, module="bs4")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
TAGS = ["NOUN", "ADJ", "VERB", "ADV"]
|
|
|
|
|
nlp = spacy.load("en_core_web_sm", disable=["parser", "ner"])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
log = util.get_logger("process")
|
|
|
|
|
|
|
|
|
|
# Maximum number of CPU threads to use for post processing
|
|
|
|
@ -49,67 +97,44 @@ hash_key = get_hash_key()
|
|
|
|
|
|
|
|
|
|
@asyncio.coroutine
|
|
|
|
|
async def spawn_processing_threads(data):
|
|
|
|
|
len_data = len(data)
|
|
|
|
|
log.debug(f"Spawning processing threads for batch of {len_data} messages")
|
|
|
|
|
|
|
|
|
|
loop = asyncio.get_event_loop()
|
|
|
|
|
tasks = []
|
|
|
|
|
oldts = [x["now"] for x in data if "now" in x]
|
|
|
|
|
|
|
|
|
|
if len(data) < CPU_THREADS:
|
|
|
|
|
split_data = [data]
|
|
|
|
|
else:
|
|
|
|
|
msg_per_core = int(len(data) / CPU_THREADS)
|
|
|
|
|
print("MSG PER CORE", msg_per_core)
|
|
|
|
|
split_data = array_split(data, ceil(len(data) / msg_per_core))
|
|
|
|
|
for index, split in enumerate(split_data):
|
|
|
|
|
print("DELEGATING TO THREAD", len(split))
|
|
|
|
|
future = loop.run_in_executor(p, process_data, data)
|
|
|
|
|
# future = p.submit(process_data, split)
|
|
|
|
|
tasks.append(future)
|
|
|
|
|
# results = [x.result(timeout=50) for x in tasks]
|
|
|
|
|
results = await asyncio.gather(*tasks)
|
|
|
|
|
print("RESULTS", len(results))
|
|
|
|
|
log.debug(f"Delegating processing of {len(split)} messages to thread {index}")
|
|
|
|
|
task = loop.run_in_executor(p, process_data, data)
|
|
|
|
|
tasks.append(task)
|
|
|
|
|
|
|
|
|
|
results = [await task for task in tasks]
|
|
|
|
|
log.debug(f"Results from processing of {len_data} messages: {len(results)}")
|
|
|
|
|
|
|
|
|
|
# Join the results back from the split list
|
|
|
|
|
flat_list = [item for sublist in results for item in sublist]
|
|
|
|
|
print("LENFLAT", len(flat_list))
|
|
|
|
|
print("LENDATA", len(data))
|
|
|
|
|
|
|
|
|
|
newts = [x["ts"] for x in flat_list if "ts" in x]
|
|
|
|
|
print("lenoldts", len(oldts))
|
|
|
|
|
print("lennewts", len(newts))
|
|
|
|
|
allts = all(["ts" in x for x in flat_list])
|
|
|
|
|
print("ALLTS", allts)
|
|
|
|
|
alllen = [len(x) for x in flat_list]
|
|
|
|
|
print("ALLLEN", alllen)
|
|
|
|
|
await db.store_kafka_batch(flat_list)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# @asyncio.coroutine
|
|
|
|
|
# def process_data_thread(data):
|
|
|
|
|
# """
|
|
|
|
|
# Helper to spawn threads to process a list of data.
|
|
|
|
|
# """
|
|
|
|
|
# loop = asyncio.get_event_loop()
|
|
|
|
|
# if len(data) < CPU_THREADS:
|
|
|
|
|
# split_data = [data]
|
|
|
|
|
# else:
|
|
|
|
|
# msg_per_core = int(len(data) / CPU_THREADS)
|
|
|
|
|
# print("MSG PER CORE", msg_per_core)
|
|
|
|
|
# split_data = array_split(data, ceil(len(data) / msg_per_core))
|
|
|
|
|
# for index, split in enumerate(split_data):
|
|
|
|
|
# print("DELEGATING TO THREAD", len(split))
|
|
|
|
|
# #f = process_data_thread(split)
|
|
|
|
|
# yield loop.run_in_executor(p, process_data, data)
|
|
|
|
|
log.debug(f"Finished processing {len_data} messages")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def process_data(data):
|
|
|
|
|
print("PROCESS DATA START")
|
|
|
|
|
# to_store = []
|
|
|
|
|
for index, msg in enumerate(data):
|
|
|
|
|
# print("PROCESSING", msg)
|
|
|
|
|
to_store = []
|
|
|
|
|
|
|
|
|
|
# Initialise sentiment analyser
|
|
|
|
|
analyzer = SentimentIntensityAnalyzer()
|
|
|
|
|
for msg in data:
|
|
|
|
|
if msg["src"] == "4ch":
|
|
|
|
|
board = msg["net"]
|
|
|
|
|
thread = msg["channel"]
|
|
|
|
|
|
|
|
|
|
# Calculate hash for post
|
|
|
|
|
post_normalised = ujson.dumps(msg, sort_keys=True)
|
|
|
|
|
post_normalised = orjson.dumps(msg, option=orjson.OPT_SORT_KEYS)
|
|
|
|
|
hash = siphash(hash_key, post_normalised)
|
|
|
|
|
hash = str(hash)
|
|
|
|
|
redis_key = f"cache.{board}.{thread}.{msg['no']}"
|
|
|
|
@ -117,19 +142,17 @@ def process_data(data):
|
|
|
|
|
if key_content:
|
|
|
|
|
key_content = key_content.decode("ascii")
|
|
|
|
|
if key_content == hash:
|
|
|
|
|
del data[index]
|
|
|
|
|
# This deletes the message since the append at the end won't be hit
|
|
|
|
|
continue
|
|
|
|
|
else:
|
|
|
|
|
data[index]["type"] = "update"
|
|
|
|
|
msg["type"] = "update"
|
|
|
|
|
db.r.set(redis_key, hash)
|
|
|
|
|
if "now" not in data[index]:
|
|
|
|
|
print("NOW NOT IN INDEX", data[index])
|
|
|
|
|
for key2, value in list(data[index].items()):
|
|
|
|
|
for key2, value in list(msg.items()):
|
|
|
|
|
if key2 in ATTRMAP:
|
|
|
|
|
data[index][ATTRMAP[key2]] = data[index][key2]
|
|
|
|
|
del data[index][key2]
|
|
|
|
|
if "ts" in data[index]:
|
|
|
|
|
old_time = data[index]["ts"]
|
|
|
|
|
msg[ATTRMAP[key2]] = msg[key2]
|
|
|
|
|
del msg[key2]
|
|
|
|
|
if "ts" in msg:
|
|
|
|
|
old_time = msg["ts"]
|
|
|
|
|
# '08/30/22(Tue)02:25:37'
|
|
|
|
|
time_spl = old_time.split(":")
|
|
|
|
|
if len(time_spl) == 3:
|
|
|
|
@ -138,14 +161,42 @@ def process_data(data):
|
|
|
|
|
old_ts = datetime.strptime(old_time, "%m/%d/%y(%a)%H:%M")
|
|
|
|
|
# new_ts = old_ts.isoformat()
|
|
|
|
|
new_ts = int(old_ts.timestamp())
|
|
|
|
|
data[index]["ts"] = new_ts
|
|
|
|
|
msg["ts"] = new_ts
|
|
|
|
|
else:
|
|
|
|
|
print("MSG WITHOUT TS PROCESS", data[index])
|
|
|
|
|
continue
|
|
|
|
|
raise Exception("No TS in msg")
|
|
|
|
|
if "msg" in msg:
|
|
|
|
|
soup = BeautifulSoup(data[index]["msg"], "html.parser")
|
|
|
|
|
msg = soup.get_text(separator="\n")
|
|
|
|
|
data[index]["msg"] = msg
|
|
|
|
|
# to_store.append(data[index])
|
|
|
|
|
print("FINISHED PROCESSING DATA")
|
|
|
|
|
return data
|
|
|
|
|
soup = BeautifulSoup(msg["msg"], "html.parser")
|
|
|
|
|
msg_str = soup.get_text(separator="\n")
|
|
|
|
|
msg["msg"] = msg_str
|
|
|
|
|
# Annotate sentiment/NLP
|
|
|
|
|
if "msg" in msg:
|
|
|
|
|
# Language
|
|
|
|
|
text = Text(msg["msg"])
|
|
|
|
|
try:
|
|
|
|
|
lang_code = text.language.code
|
|
|
|
|
lang_name = text.language.name
|
|
|
|
|
msg["lang_code"] = lang_code
|
|
|
|
|
msg["lang_name"] = lang_name
|
|
|
|
|
except cld2_error as e:
|
|
|
|
|
log.error(f"Error detecting language: {e}")
|
|
|
|
|
# So below block doesn't fail
|
|
|
|
|
lang_code = None
|
|
|
|
|
|
|
|
|
|
# Blatant discrimination
|
|
|
|
|
if lang_code == "en":
|
|
|
|
|
|
|
|
|
|
# Sentiment
|
|
|
|
|
vs = analyzer.polarity_scores(str(msg["msg"]))
|
|
|
|
|
addendum = vs["compound"]
|
|
|
|
|
msg["sentiment"] = addendum
|
|
|
|
|
|
|
|
|
|
# Tokens
|
|
|
|
|
n = nlp(msg["msg"])
|
|
|
|
|
for tag in TAGS:
|
|
|
|
|
tag_name = tag.lower()
|
|
|
|
|
tags_flag = [token.lemma_ for token in n if token.pos_ == tag]
|
|
|
|
|
msg[f"words_{tag_name}"] = tags_flag
|
|
|
|
|
|
|
|
|
|
# Add the mutated message to the return buffer
|
|
|
|
|
to_store.append(msg)
|
|
|
|
|
return to_store
|
|
|
|
|