from copy import deepcopy from datetime import datetime from json import dumps import main from siphashc import siphash from utils.logging.debug import debug def dedup(numName, b): c = deepcopy(b) if "ts" in c.keys(): del c["ts"] c["approxtime"] = str(datetime.utcnow().timestamp())[ : main.config["Tweaks"]["DedupPrecision"] ] castHash = siphash(main.hashKey, dumps(c, sort_keys=True)) del c["approxtime"] isDuplicate = any( castHash in main.lastEvents[x] for x in main.lastEvents.keys() if not x == numName ) if isDuplicate: debug("Duplicate: %s" % (c)) return True if numName in main.lastEvents.keys(): main.lastEvents[numName].insert(0, castHash) main.lastEvents[numName] = main.lastEvents[numName][ 0 : main.config["Tweaks"]["MaxHash"] ] # noqa else: main.lastEvents[numName] = [castHash] return False