Implement sentiment/NLP annotation and optimise processing

docker/Dockerfile

@@ -16,7 +16,7 @@ COPY requirements.txt /code/
 COPY discord-patched.tgz /code/
 
 RUN python -m venv /venv
-RUN . /venv/bin/activate && pip install -r requirements.txt
+RUN . /venv/bin/activate && pip install -r requirements.txt && python -m spacy download en_core_web_sm
 
 RUN tar xf /code/discord-patched.tgz -C /venv/lib/python3.10/site-packages
 

docker/requirements.txt

@@ -4,8 +4,18 @@ redis
 siphashc
 aiohttp[speedups]
 python-dotenv
-manticoresearch
+#manticoresearch
 numpy
 ujson
 aioredis[hiredis]
 aiokafka
+vaderSentiment
+polyglot
+pyicu
+pycld2
+morfessor
+six
+nltk
+spacy
+python-Levenshtein
+orjson