Implement indexing into Apache Druid #1
|
@ -5,5 +5,5 @@ redis
|
|||
pyYaML
|
||||
python-logstash
|
||||
service_identity
|
||||
csiphash
|
||||
siphashc
|
||||
Klein
|
||||
|
|
|
@ -6,5 +6,5 @@ redis
|
|||
pyYaML
|
||||
python-logstash
|
||||
service_identity
|
||||
csiphash
|
||||
siphashc
|
||||
Klein
|
||||
|
|
|
@ -2,7 +2,7 @@ from copy import deepcopy
|
|||
from datetime import datetime
|
||||
from json import dumps
|
||||
|
||||
from csiphash import siphash24
|
||||
from siphashc import siphash
|
||||
|
||||
import main
|
||||
from utils.logging.debug import debug
|
||||
|
@ -13,7 +13,7 @@ def dedup(numName, b):
|
|||
if "ts" in c.keys():
|
||||
del c["ts"]
|
||||
c["approxtime"] = str(datetime.utcnow().timestamp())[: main.config["Tweaks"]["DedupPrecision"]]
|
||||
castHash = siphash24(main.hashKey, dumps(c, sort_keys=True).encode("utf-8"))
|
||||
castHash = siphash(main.hashKey, dumps(c, sort_keys=True))
|
||||
del c["approxtime"]
|
||||
isDuplicate = any(castHash in main.lastEvents[x] for x in main.lastEvents.keys() if not x == numName)
|
||||
if isDuplicate:
|
||||
|
|
Loading…
Reference in New Issue