from copy import deepcopy from datetime import datetime from json import dumps from siphashc import siphash import main from utils.logging.debug import debug def dedup(numName, b): c = deepcopy(b) if "ts" in c.keys(): del c["ts"] c["approxtime"] = str(datetime.utcnow().timestamp())[: main.config["Tweaks"]["DedupPrecision"]] castHash = siphash(main.hashKey, dumps(c, sort_keys=True)) del c["approxtime"] isDuplicate = any(castHash in main.lastEvents[x] for x in main.lastEvents.keys() if not x == numName) if isDuplicate: debug("Duplicate: %s" % (c)) return True if numName in main.lastEvents.keys(): main.lastEvents[numName].insert(0, castHash) main.lastEvents[numName] = main.lastEvents[numName][0 : main.config["Tweaks"]["MaxHash"]] # noqa else: main.lastEvents[numName] = [castHash] return False