Merge branch 'master' into datarestructure

utils/dedup.py

@@ -0,0 +1,21 @@
+from datetime import datetime
+from csiphash import siphash24
+from json import dumps
+import main
+from utils.logging.debug import debug
+
+def dedup(numName, c):
+    # deduplication
+    c["approxtime"] = str(datetime.utcnow().timestamp())[:main.config["Tweaks"]["DedupPrecision"]]
+    castHash = siphash24(main.hashKey, dumps(c, sort_keys=True).encode("utf-8"))
+    del c["approxtime"]
+    isDuplicate= any(castHash in main.lastEvents[x] for x in main.lastEvents.keys() if not x == numName)
+    if isDuplicate:
+        debug("Duplicate: %s" % (c))
+        return True
+    if numName in main.lastEvents.keys():
+        main.lastEvents[numName].insert(0, castHash)
+        main.lastEvents[numName] = main.lastEvents[numName][0:main.config["Tweaks"]["MaxHash"]]
+    else:
+        main.lastEvents[numName] = [castHash]
+    return False

utils/loaders/command_loader.py

@@ -1,6 +1,6 @@
 from os import listdir
 
-from utils.logging.log import *
+from utils.logging.debug import debug
 import commands
 
 from main import CommandMap

utils/logging/debug.py

@@ -0,0 +1,7 @@
+import main
+# we need a seperate module to log.py, as log.py is imported by main.py, and we need to access main
+#  to read the setting
+def debug(data):
+    if main.config["Debug"]:
+        print("[DEBUG]", data)
+

utils/logging/log.py

@@ -1,9 +1,6 @@
 def log(data):
     print("[LOG]", data)
 
-def debug(data):
-    print("[DEBUG]", data)
-
 def warn(data):
     print("[WARNING]", data)