Normalise fields in processing and remove invalid characters

2022-09-21 10:01:12 +01:00
parent 740f93208b
commit ced3a251b2
4 changed files with 33 additions and 11 deletions
--- a/sources/ingest.py
+++ b/sources/ingest.py
@@ -8,6 +8,10 @@ import util
 from processing import process

 SOURCES = ["4ch", "irc", "dis"]
+# DEBUG CODE REMOVE ME
+SOURCES.remove("4ch")
+SOURCES.remove("dis")
+# DEBUG CODE REMOVE ME
 KEYPREFIX = "queue."

 # Chunk size per source (divide by len(SOURCES) for total)