Improvements to query and self event detection, implement all command and debug flags

utils/dedup.py

@@ -0,0 +1,21 @@
+from datetime import datetime
+from csiphash import siphash24
+from json import dumps
+import main
+from utils.logging.debug import debug
+
+def dedup(numName, c):
+    # deduplication
+    c["approxtime"] = int(datetime.utcnow().timestamp())
+    castHash = siphash24(main.hashKey, dumps(c, sort_keys=True).encode("utf-8"))
+    del c["approxtime"]
+    isDuplicate= any(castHash in main.lastEvents[x] for x in main.lastEvents.keys() if not x == numName)
+    if isDuplicate:
+        debug("Duplicate: %s" % (c))
+        return True
+    if numName in main.lastEvents.keys():
+        main.lastEvents[numName].insert(0, castHash)
+        main.lastEvents[numName] = main.lastEvents[numName][0:main.config["Tweaks"]["MaxHash"]]
+    else:
+        main.lastEvents[numName] = [castHash]
+    return False