From f34ddab6fc5b8eeecc47b4dd477926a9bc05447c Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 15 Aug 2019 21:20:49 +0100 Subject: [PATCH] Improvements to query and self event detection, implement all command and debug flags --- commands/all.py | 23 ++++++++++++++++++++++ conf/example/config.json | 2 ++ conf/help.json | 3 ++- core/bot.py | 35 +++++++++++++++++++-------------- main.py | 6 ++++++ modules/monitor.py | 12 ++++++++--- modules/provision.py | 3 ++- modules/userinfo.py | 7 +++++++ threshold | 4 +++- utils/dedup.py | 21 ++++++++++++++++++++ utils/loaders/command_loader.py | 2 +- utils/logging/debug.py | 7 +++++++ utils/logging/log.py | 3 --- 13 files changed, 103 insertions(+), 25 deletions(-) create mode 100644 commands/all.py create mode 100644 utils/dedup.py create mode 100644 utils/logging/debug.py diff --git a/commands/all.py b/commands/all.py new file mode 100644 index 0000000..10ec381 --- /dev/null +++ b/commands/all.py @@ -0,0 +1,23 @@ +import main +from core.bot import deliverRelayCommands + +class All: + def __init__(self, *args): + self.all(*args) + + def all(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + if authed: + if length > 2: + for i in main.pool.keys(): + relay = main.pool[i]["relay"] + network = main.pool[i]["network"] + alias = main.pool[i]["alias"] + commands = {spl[1]: [" ".join(spl[2:])]} + success("Sending commands to relay %s as user %s" % (relay, alias+"/"+network)) + deliverRelayCommands(relay, commands, user=alias+"/"+network) + return + else: + incUsage("all") + return + else: + incUsage(None) diff --git a/conf/example/config.json b/conf/example/config.json index bc67bdf..5ecb53f 100644 --- a/conf/example/config.json +++ b/conf/example/config.json @@ -15,6 +15,7 @@ "RedisSocket": "/tmp/redis.sock", "UsePassword": true, "ConnectOnCreate": false, + "Debug": false, "Dist": { "Enabled": true, "SendOutput": false, @@ -26,6 +27,7 @@ }, "Password": "s", "Tweaks": { + "MaxHash": 10, "ZNC": { "Prefix": "*" }, diff --git a/conf/help.json b/conf/help.json index 0211171..522f00e 100644 --- a/conf/help.json +++ b/conf/help.json @@ -24,5 +24,6 @@ "network": "network [
]", "provision": "provision []", "cmd": "cmd ", - "token": "token [] []" + "token": "token [] []", + "all": "all " } diff --git a/core/bot.py b/core/bot.py index ae4a0fe..33a20e9 100644 --- a/core/bot.py +++ b/core/bot.py @@ -9,10 +9,11 @@ from random import randint from copy import deepcopy from modules import userinfo -from modules import counters as count +from modules import counters from modules import monitor from core.relay import sendRelayNotification +from utils.dedup import dedup import main from utils.logging.log import * @@ -151,9 +152,9 @@ class IRCBot(IRCClient): del cast["host"] del cast["target"] if not cast["type"] in ["query", "self", "highlight", "znc", "who"]: - if "target" in cast.keys(): - if cast["target"].lower() == self.nickname.lower(): - #castDup = deepcopy(cast) + if "target" in cast.keys() and not cast["type"] == "mode": # don't handle modes here + if cast["target"].lower() == self.nickname.lower(): # as they are target == nickname + #castDup = deepcopy(cast) # however modes are not queries! cast["mtype"] = cast["type"] cast["type"] = "query" cast["name"] = self.name @@ -173,7 +174,8 @@ class IRCBot(IRCClient): castDup["mtype"] = cast["type"] castDup["type"] = "self" castDup["name"] = self.name - self.event(**castDup) + if not cast["target"].lower() == self.nickname.lower(): # modes has been set on us directly + self.event(**castDup) # don't tell anyone else if "message" in cast.keys() and not cast["type"] == "query": # Don't highlight queries if self.nickname.lower() in cast["message"].lower(): castDup = deepcopy(cast) @@ -184,8 +186,8 @@ class IRCBot(IRCClient): if not "name" in cast.keys(): cast["name"] = self.net - count.event(self.net, cast["type"]) - monitor.event(cast) + counters.event(self.net, cast["type"]) + monitor.event(self.name, cast) def privmsg(self, user, channel, msg): self.event(type="msg", muser=user, target=channel, message=msg) @@ -408,37 +410,40 @@ class IRCBot(IRCClient): self.event(type="part", muser=user, target=channel, message=message) def userQuit(self, user, quitMessage): - self.chanlessEvent(type="quit", muser=user, message=quitMessage) + self.chanlessEvent({"type": "quit", "muser": user, "message": quitMessage}) def userKicked(self, kickee, channel, kicker, message): if kickee.lower() == self.nickname.lower(): self.botLeft(channel) self.event(type="kick", muser=kicker, target=channel, message=message, user=kickee) - def chanlessEvent(self, **cast): - chans = userinfo.getChansSingle(self.net, cast["nick"]) + def chanlessEvent(self, cast): + cast["nick"], cast["ident"], cast["host"] = self.parsen(cast["muser"]) + if dedup(self.name, cast): + return # stop right there sir! + chans = userinfo.getChanList(self.net, cast["nick"]) if chans == None: - self.event(**cast) + error("No channels returned for chanless event: %s" % cast) + # self.event(**cast) -- no, should NEVER happen return # getChansSingle returns all channels of the user, we only want to use # ones we have common with them realChans = set(chans).intersection(set(self.channels)) - for i in chans: + for i in realChans: cast["target"] = i self.event(**cast) def userRenamed(self, oldname, newname): - self.chanlessEvent(type="nick", muser=oldname, user=newname) + self.chanlessEvent({"type": "nick", "muser": oldname, "user": newname}) def topicUpdated(self, user, channel, newTopic): self.event(type="topic", muser=user, target=channel, message= newTopic) def modeChanged(self, user, channel, toset, modes, args): - nick, ident, host = self.parsen(user) argList = list(args) modeList = [i for i in modes] for a, m in zip(argList, modeList): - self.event(type="mode", nick=nick, ident=ident, host=host, target=channel, modes=m, status=toset, modeargs=a) + self.event(type="mode", muser=user, target=channel, modes=m, status=toset, modeargs=a) class IRCBotFactory(ReconnectingClientFactory): def __init__(self, name, relay=None, relayCommands=None, user=None, stage2=None): diff --git a/main.py b/main.py index 1aa0cdf..ccb6942 100644 --- a/main.py +++ b/main.py @@ -1,6 +1,8 @@ from json import load, dump, loads from redis import StrictRedis from string import digits +from os import urandom + from utils.logging.log import * configPath = "conf/" @@ -31,6 +33,10 @@ CommandMap = {} runningSample = 0 lastMinuteSample = 0 +# Generate 16-byte hex key for message checksums +hashKey = urandom(16) +lastEvents = {} + def nets(): if not "pool" in globals(): return diff --git a/modules/monitor.py b/modules/monitor.py index 16b0bb4..3913c6f 100644 --- a/modules/monitor.py +++ b/modules/monitor.py @@ -1,9 +1,11 @@ from copy import deepcopy from json import dumps +from datetime import datetime import main from core.relay import sendRelayNotification from modules import userinfo +from utils.dedup import dedup def testNetTarget(name, target): called = False @@ -51,10 +53,12 @@ def magicFunction(A, B): else: return all(A[k] in B[k] for k in set(A) & set(B)) and set(B) <= set(A) -def event(c): # yes I'm using a short variable because otherwise it goes off the screen +def event(numName, c): # yes I'm using a short variable because otherwise it goes off the screen if not "target" in c.keys(): c["target"] = None + if dedup(numName, c): + return # metadata scraping # need to check if this was received from a relay # in which case, do not do this @@ -68,9 +72,9 @@ def event(c): # yes I'm using a short variable because otherwise it goes off the elif c["type"] == "quit": userinfo.delUserByNetwork(c["name"], c["nick"], c["muser"]) elif c["type"] == "join": - userinfo.addUser(c["name"], c["target"], c["nick"], c["user"]) + userinfo.addUser(c["name"], c["target"], c["nick"], c["muser"]) elif c["type"] == "part": - userinfo.delUser(c["name"], c["target"], c["nick"], c["user"]) + userinfo.delUser(c["name"], c["target"], c["nick"], c["muser"]) if "mtype" in c.keys(): if c["mtype"] == "nick": @@ -79,6 +83,8 @@ def event(c): # yes I'm using a short variable because otherwise it goes off the if "muser" in c.keys(): del c["muser"] sendRelayNotification(c) + + # only monitors below monitorGroups = testNetTarget(c["name"], c["target"]) if monitorGroups == False: return diff --git a/modules/provision.py b/modules/provision.py index f964cf3..fb3ea9c 100644 --- a/modules/provision.py +++ b/modules/provision.py @@ -34,7 +34,7 @@ def provisionNetworkData(relay, alias, network, host, port, security, auth, pass elif auth == "ns": stage2commands["status"] = [] stage2commands["nickserv"] = [] - stage2commands["status"].append("LoadMod NickServ") + stage2commands["status"].append("LoadMod nickserv") stage2commands["nickserv"].append("Set %s" % password) if not main.config["ConnectOnCreate"]: stage3commands["status"] = [] @@ -42,6 +42,7 @@ def provisionNetworkData(relay, alias, network, host, port, security, auth, pass if main.config["Toggles"]["CycleChans"]: stage2commands["status"] = [] stage2commands["status"].append("LoadMod disconkick") + stage2commands["status"].append("LoadMod chansaver") deliverRelayCommands(relay, commands, stage2=[[alias+"/"+network, stage2commands], [alias+"/"+network, stage3commands]]) diff --git a/modules/userinfo.py b/modules/userinfo.py index d9bdddf..00020cd 100644 --- a/modules/userinfo.py +++ b/modules/userinfo.py @@ -23,6 +23,13 @@ def getChansSingle(name, nick): return None return [i.decode() for i in result] +def getChanList(name, nick): + chanspace = "live.chan."+name+"."+nick + result = main.r.smembers(chanspace) + if len(result) == 0: + return None + return [i.decode() for i in result] + def getChans(nick): result = {} for i in main.nets(): diff --git a/threshold b/threshold index 5ff3a3f..648ed4f 100755 --- a/threshold +++ b/threshold @@ -1,6 +1,7 @@ #!/usr/bin/env python from twisted.internet import reactor from twisted.internet.ssl import DefaultOpenSSLContextFactory +from sys import argv #from twisted.python import log #from sys import stdout #log.startLogging(stdout) @@ -8,7 +9,8 @@ from twisted.internet.ssl import DefaultOpenSSLContextFactory import main main.initMain() - +if "--debug" in argv: # yes really + main.config["Debug"] = True from utils.logging.log import * from utils.loaders.command_loader import loadCommands from core.helper import startBot diff --git a/utils/dedup.py b/utils/dedup.py new file mode 100644 index 0000000..ee07bb5 --- /dev/null +++ b/utils/dedup.py @@ -0,0 +1,21 @@ +from datetime import datetime +from csiphash import siphash24 +from json import dumps +import main +from utils.logging.debug import debug + +def dedup(numName, c): + # deduplication + c["approxtime"] = int(datetime.utcnow().timestamp()) + castHash = siphash24(main.hashKey, dumps(c, sort_keys=True).encode("utf-8")) + del c["approxtime"] + isDuplicate= any(castHash in main.lastEvents[x] for x in main.lastEvents.keys() if not x == numName) + if isDuplicate: + debug("Duplicate: %s" % (c)) + return True + if numName in main.lastEvents.keys(): + main.lastEvents[numName].insert(0, castHash) + main.lastEvents[numName] = main.lastEvents[numName][0:main.config["Tweaks"]["MaxHash"]] + else: + main.lastEvents[numName] = [castHash] + return False diff --git a/utils/loaders/command_loader.py b/utils/loaders/command_loader.py index e3ea253..ad623cc 100644 --- a/utils/loaders/command_loader.py +++ b/utils/loaders/command_loader.py @@ -1,6 +1,6 @@ from os import listdir -from utils.logging.log import * +from utils.logging.debug import debug import commands from main import CommandMap diff --git a/utils/logging/debug.py b/utils/logging/debug.py new file mode 100644 index 0000000..5a7b067 --- /dev/null +++ b/utils/logging/debug.py @@ -0,0 +1,7 @@ +import main +# we need a seperate module to log.py, as log.py is imported by main.py, and we need to access main +# to read the setting +def debug(data): + if main.config["Debug"]: + print("[DEBUG]", data) + diff --git a/utils/logging/log.py b/utils/logging/log.py index 5e1141f..6bb6a99 100644 --- a/utils/logging/log.py +++ b/utils/logging/log.py @@ -1,9 +1,6 @@ def log(data): print("[LOG]", data) -def debug(data): - print("[DEBUG]", data) - def warn(data): print("[WARNING]", data)