monolith/utils/dedup.py

from copy import deepcopy
from datetime import datetime
from json import dumps

from csiphash import siphash24

import main
from utils.logging.debug import debug


def dedup(numName, b):
    c = deepcopy(b)
    if "ts" in c.keys():
        del c["ts"]
    c["approxtime"] = str(datetime.utcnow().timestamp())[
        : main.config["Tweaks"]["DedupPrecision"]
    ]
    castHash = siphash24(main.hashKey, dumps(c, sort_keys=True).encode("utf-8"))
    del c["approxtime"]
    isDuplicate = any(
        castHash in main.lastEvents[x]
        for x in main.lastEvents.keys()
        if not x == numName
    )
    if isDuplicate:
        debug("Duplicate: %s" % (c))
        return True
    if numName in main.lastEvents.keys():
        main.lastEvents[numName].insert(0, castHash)
        main.lastEvents[numName] = main.lastEvents[numName][
            0 : main.config["Tweaks"]["MaxHash"]
        ]
    else:
        main.lastEvents[numName] = [castHash]
    return False