22 lines
786 B
Python
22 lines
786 B
Python
|
from datetime import datetime
|
||
|
|
from csiphash import siphash24
|
||
|
|
from json import dumps
|
||
|
|
import main
|
||
|
|
from utils.logging.debug import debug
|
||
|
|
|
||
|
|
def dedup(numName, c):
|
||
|
|
# deduplication
|
||
|
|
c["approxtime"] = int(datetime.utcnow().timestamp())
|
||
|
|
castHash = siphash24(main.hashKey, dumps(c, sort_keys=True).encode("utf-8"))
|
||
|
|
del c["approxtime"]
|
||
|
|
isDuplicate= any(castHash in main.lastEvents[x] for x in main.lastEvents.keys() if not x == numName)
|
||
|
|
if isDuplicate:
|
||
|
|
debug("Duplicate: %s" % (c))
|
||
|
|
return True
|
||
|
|
if numName in main.lastEvents.keys():
|
||
|
|
main.lastEvents[numName].insert(0, castHash)
|
||
|
|
main.lastEvents[numName] = main.lastEvents[numName][0:main.config["Tweaks"]["MaxHash"]]
|
||
|
|
else:
|
||
|
|
main.lastEvents[numName] = [castHash]
|
||
|
|
return False
|