From 51b89b9d0590ccc6a6f6bdbf548473872cd14d56 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Sun, 11 Aug 2019 22:05:34 +0100 Subject: [PATCH 1/6] Use the ISO format for time --- core/relay.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/relay.py b/core/relay.py index 765b783..58ba3d5 100644 --- a/core/relay.py +++ b/core/relay.py @@ -138,5 +138,5 @@ def sendRelayNotification(name, cast): if cast["type"] in main.relayConnections[i].subscriptions: newCast = deepcopy(cast) newCast["name"] = name - newCast["time"] = str(datetime.now()) + newCast["time"] = str(datetime.now().isoformat()) main.relayConnections[i].send(dumps(newCast)) From 1ec0e1f7e608bf71e82c08660f807e978184eed1 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Mon, 12 Aug 2019 21:03:47 +0100 Subject: [PATCH 2/6] Remove provisioning restrictions, move all user tracking code to monitoring module, fix proper network name not being passed to the relay --- commands/provision.py | 36 +++++++++++++-------------- core/bot.py | 57 ++++++++++++++++--------------------------- core/relay.py | 3 +-- modules/monitor.py | 48 ++++++++++++++++++++++++++---------- modules/provision.py | 30 +++++++++++------------ 5 files changed, 90 insertions(+), 84 deletions(-) diff --git a/commands/provision.py b/commands/provision.py index 3453171..7803b6f 100644 --- a/commands/provision.py +++ b/commands/provision.py @@ -19,22 +19,22 @@ class Provision: failure("No such network: %s" % spl[3]) return - if "users" in main.relay[spl[1]]: - if not spl[2] in main.relay[spl[1]]["users"]: - failure("Relay %s not provisioned for alias %s" % (spl[1], spl[2])) - return - else: - failure("Relay %s not provisioned for alias %s" % (spl[1], spl[2])) - return + #if "users" in main.relay[spl[1]]: + # if not spl[2] in main.relay[spl[1]]["users"]: + # failure("Relay %s not provisioned for alias %s" % (spl[1], spl[2])) + # return + #else: + # failure("Relay %s not provisioned for alias %s" % (spl[1], spl[2])) + # return rtrn = provision.provisionRelayForNetwork(spl[1], spl[2], spl[3]) - if rtrn == "PROVISIONED": - failure("Relay %s already provisioned for alias %s on network %s" % (spl[1], spl[2], spl[3])) - return - elif rtrn == "DUPLICATE": - failure("Instance with relay %s and alias %s already exists for network %s" % (spl[1], spl[2], spl[3])) - return - elif rtrn: + #if rtrn == "PROVISIONED": + # failure("Relay %s already provisioned for alias %s on network %s" % (spl[1], spl[2], spl[3])) + # return + #elif rtrn == "DUPLICATE": + # failure("Instance with relay %s and alias %s already exists for network %s" % (spl[1], spl[2], spl[3])) + # return + if rtrn: success("Started provisioning network %s on relay %s for alias %s" % (spl[3], spl[1], spl[2])) info("Instance name is %s" % rtrn) return @@ -43,10 +43,10 @@ class Provision: return if length == 3: # provision for relay and alias only rtrn = provision.provisionRelayForAlias(spl[1], spl[2]) - if rtrn == "PROVISIONED": - failure("Relay %s already provisioned for alias %s" % (spl[1], spl[2])) - return - elif rtrn: + #if rtrn == "PROVISIONED": + # failure("Relay %s already provisioned for alias %s" % (spl[1], spl[2])) + # return + if rtrn: success("Started provisioning relay %s for alias %s" % (spl[1], spl[2])) return else: diff --git a/core/bot.py b/core/bot.py index ee5d8b7..ae4a0fe 100644 --- a/core/bot.py +++ b/core/bot.py @@ -139,7 +139,8 @@ class IRCBot(IRCClient): del cast[i] if "muser" in cast.keys(): cast["nick"], cast["ident"], cast["host"] = self.parsen(cast["muser"]) - del cast["muser"] + #if not cast["type"] in ["nick", "kick", "quit", "part", "join"]: + # del cast["muser"] if set(["nick", "ident", "host", "message"]).issubset(set(cast)): if "message" in cast.keys(): if cast["ident"] == "znc" and cast["host"] == "znc.in": @@ -181,12 +182,10 @@ class IRCBot(IRCClient): castDup["name"] = self.name self.event(**castDup) - if "name" not in cast.keys(): + if not "name" in cast.keys(): cast["name"] = self.net - if cast["type"] in ["msg", "notice", "action"]: - userinfo.editUser(self.net, cast["nick"]+"!"+cast["ident"]+"@"+cast["host"]) count.event(self.net, cast["type"]) - monitor.event(self.net, self.name, cast) + monitor.event(cast) def privmsg(self, user, channel, msg): self.event(type="msg", muser=user, target=channel, message=msg) @@ -214,10 +213,8 @@ class IRCBot(IRCClient): return newnick def nickChanged(self, olduser, newnick): - oldnick, ident, host = self.parsen(olduser) - userinfo.renameUser(self.net, oldnick, olduser, newnick, newnick+"!"+ident+"@"+host) self.nickname = newnick - self.event(type="self", mtype="nick", nick=oldnick, ident=ident, host=host, user=newnick) + self.event(type="self", mtype="nick", muser=olduser, user=newnick) def irc_ERR_NICKNAMEINUSE(self, prefix, params): self._attemptedNick = self.alterCollidedNick(self._attemptedNick) @@ -397,59 +394,47 @@ class IRCBot(IRCClient): lc = self._getWho[channel] lc.stop() del self._getWho[channel] - userinfo.delChannel(self.net, channel) - log("Can no longer cover %s, removing records" % channel) - - def left(self, user, channel, message): + userinfo.delChannel(self.net, channel) # < we do not need to deduplicate this + log("Can no longer cover %s, removing records" % channel) # as it will only be matched once -- + # other bots have different nicknames so + def left(self, user, channel, message): # even if they saw it, they wouldn't react self.event(type="part", muser=user, target=channel, message=message) self.botLeft(channel) def userJoined(self, user, channel): - nick, ident, host = self.parsen(user) - userinfo.addUser(self.net, channel, nick, user) - self.event(type="join", nick=nick, ident=ident, host=host, target=channel) + self.event(type="join", muser=user, target=channel) def userLeft(self, user, channel, message): - nick, ident, host = self.parsen(user) - userinfo.delUser(self.net, channel, nick, user) - self.event(type="part", nick=nick, ident=ident, host=host, target=channel, message=message) + self.event(type="part", muser=user, target=channel, message=message) def userQuit(self, user, quitMessage): - nick, ident, host = self.parsen(user) - self.chanlessEvent(type="quit", nick=nick, ident=ident, host=host, message=quitMessage) - userinfo.delUserByNetwork(self.net, nick, user) + self.chanlessEvent(type="quit", muser=user, message=quitMessage) def userKicked(self, kickee, channel, kicker, message): - nick, ident, host = self.parsen(kicker) - userinfo.editUser(self.net, kicker) - userinfo.delUserByNick(self.net, channel, kickee) - if kickee.lower == self.nickname.lower: + if kickee.lower() == self.nickname.lower(): self.botLeft(channel) - self.event(type="kick", nick=nick, ident=ident, host=host, target=channel, message=message, user=kickee) + self.event(type="kick", muser=kicker, target=channel, message=message, user=kickee) def chanlessEvent(self, **cast): chans = userinfo.getChansSingle(self.net, cast["nick"]) if chans == None: self.event(**cast) return + # getChansSingle returns all channels of the user, we only want to use + # ones we have common with them + realChans = set(chans).intersection(set(self.channels)) for i in chans: cast["target"] = i self.event(**cast) def userRenamed(self, oldname, newname): - nick, ident, host = self.parsen(oldname) - self.chanlessEvent(type="nick", nick=nick, ident=ident, host=host, user=newname) - userinfo.renameUser(self.net, nick, oldname, newname, newname+"!"+ident+"@"+host) + self.chanlessEvent(type="nick", muser=oldname, user=newname) def topicUpdated(self, user, channel, newTopic): - nick, ident, host = self.parsen(user) - userinfo.editUser(self.net, user) - - self.event(type="topic", nick=nick, ident=ident, host=host, target=channel, message= newTopic) + self.event(type="topic", muser=user, target=channel, message= newTopic) def modeChanged(self, user, channel, toset, modes, args): nick, ident, host = self.parsen(user) - userinfo.editUser(self.net, user) argList = list(args) modeList = [i for i in modes] for a, m in zip(argList, modeList): @@ -490,7 +475,7 @@ class IRCBotFactory(ReconnectingClientFactory): log("%s: connection lost: %s" % (self.name, error)) if not self.relay: sendAll("%s: connection lost: %s" % (self.name, error)) - sendRelayNotification(self.name, {"type": "conn", "status": "lost", "message": error}) + sendRelayNotification({"type": "conn", "name": self.name, "status": "lost", "message": error}) self.retry(connector) #ReconnectingClientFactory.clientConnectionLost(self, connector, reason) @@ -502,7 +487,7 @@ class IRCBotFactory(ReconnectingClientFactory): log("%s: connection failed: %s" % (self.name, error)) if not self.relay: sendAll("%s: connection failed: %s" % (self.name, error)) - sendRelayNotification(self.name, {"type": "conn", "status": "failed", "message": error}) + sendRelayNotification({"type": "conn", "name": self.name, "status": "failed", "message": error}) self.retry(connector) #ReconnectingClientFactory.clientConnectionFailed(self, connector, reason) diff --git a/core/relay.py b/core/relay.py index 58ba3d5..4067ee0 100644 --- a/core/relay.py +++ b/core/relay.py @@ -132,11 +132,10 @@ class RelayFactory(Factory): else: return -def sendRelayNotification(name, cast): +def sendRelayNotification(cast): for i in main.relayConnections.keys(): if main.relayConnections[i].authed: if cast["type"] in main.relayConnections[i].subscriptions: newCast = deepcopy(cast) - newCast["name"] = name newCast["time"] = str(datetime.now().isoformat()) main.relayConnections[i].send(dumps(newCast)) diff --git a/modules/monitor.py b/modules/monitor.py index 08fae94..16b0bb4 100644 --- a/modules/monitor.py +++ b/modules/monitor.py @@ -3,6 +3,7 @@ from json import dumps import main from core.relay import sendRelayNotification +from modules import userinfo def testNetTarget(name, target): called = False @@ -50,28 +51,49 @@ def magicFunction(A, B): else: return all(A[k] in B[k] for k in set(A) & set(B)) and set(B) <= set(A) -def event(name, numberedName, cast, event=None): - if "target" in cast.keys(): - target = cast["target"] - else: - target = None +def event(c): # yes I'm using a short variable because otherwise it goes off the screen + if not "target" in c.keys(): + c["target"] = None + + # metadata scraping + # need to check if this was received from a relay + # in which case, do not do this + if c["type"] in ["msg", "notice", "action", "topic", "mode"]: + userinfo.editUser(c["name"], c["muser"]) + elif c["type"] == "nick": + userinfo.renameUser(c["name"], c["nick"], c["muser"], c["user"], c["user"]+"!"+c["ident"]+"@"+c["host"]) + elif c["type"] == "kick": + userinfo.editUser(c["name"], c["muser"]) + userinfo.delUserByNick(c["name"], c["target"], c["user"]) + elif c["type"] == "quit": + userinfo.delUserByNetwork(c["name"], c["nick"], c["muser"]) + elif c["type"] == "join": + userinfo.addUser(c["name"], c["target"], c["nick"], c["user"]) + elif c["type"] == "part": + userinfo.delUser(c["name"], c["target"], c["nick"], c["user"]) + + if "mtype" in c.keys(): + if c["mtype"] == "nick": + userinfo.renameUser(c["name"], c["nick"], c["muser"], c["user"], c["user"]+"!"+c["ident"]+"@"+c["host"]) - sendRelayNotification(name, cast) - monitorGroups = testNetTarget(name, target) + if "muser" in c.keys(): + del c["muser"] + sendRelayNotification(c) + monitorGroups = testNetTarget(c["name"], c["target"]) if monitorGroups == False: return for monitorGroup in monitorGroups: - matcher = magicFunction(deepcopy(cast), deepcopy(main.monitor[monitorGroup])) + matcher = magicFunction(deepcopy(c), deepcopy(main.monitor[monitorGroup])) if matcher == True: - cast["monitor"] = True + c["monitor"] = True if "send" in main.monitor[monitorGroup].keys(): for i in main.monitor[monitorGroup]["send"].keys(): if isinstance(main.monitor[monitorGroup]["send"][i], bool): - sendRelayNotification(name, {"type": "err", "name": name, "target": target, "message": cast, "reason": "errdeliv"}) + sendRelayNotification({"type": "err", "name": name, "target": target, "message": c, "reason": "errdeliv"}) continue if not i in main.pool.keys(): - sendRelayNotification(name, {"type": "err", "name": name, "target": target, "message": cast, "reason": "noname"}) + sendRelayNotification({"type": "err", "name": name, "target": target, "message": c, "reason": "noname"}) if not i in main.IRCPool.keys(): - sendRelayNotification(name, {"type": "err", "name": name, "target": target, "message": cast, "reason": "noinstance"}) + sendRelayNotification({"type": "err", "name": name, "target": target, "message": c, "reason": "noinstance"}) for x in main.monitor[monitorGroup]["send"][i]: - main.IRCPool[i].msg(x, "monitor [%s] (%s) %s" % (monitorGroup, name, cast)) + main.IRCPool[i].msg(x, "monitor [%s] (%s) %s" % (monitorGroup, c["name"], c)) diff --git a/modules/provision.py b/modules/provision.py index 0eb7464..f964cf3 100644 --- a/modules/provision.py +++ b/modules/provision.py @@ -48,40 +48,40 @@ def provisionNetworkData(relay, alias, network, host, port, security, auth, pass return def provisionRelayForAlias(relay, alias): - if "users" in main.relay[relay].keys(): - if alias in main.relay[relay]["users"]: - return "PROVISIONED" - else: - main.relay[relay]["users"] = [] - main.relay[relay]["users"].append(alias) + #if "users" in main.relay[relay].keys(): + # if alias in main.relay[relay]["users"]: + # return "PROVISIONED" + #else: + # main.relay[relay]["users"] = [] + #main.relay[relay]["users"].append(alias) provisionUserData(relay, alias, main.alias[alias]["nick"], main.alias[alias]["altnick"], main.alias[alias]["ident"], main.alias[alias]["realname"], main.relay[relay]["password"]) - main.saveConf("relay") + #main.saveConf("relay") return True def provisionRelayForNetwork(relay, alias, network): - if set(["users", "networks"]).issubset(main.relay[relay].keys()): - if network in main.relay[relay]["networks"] and alias in main.relay[relay]["users"]: - return "PROVISIONED" - else: - main.relay[relay]["networks"] = [] - main.relay[relay]["networks"].append(network) + #if set(["users", "networks"]).issubset(main.relay[relay].keys()): + # if network in main.relay[relay]["networks"] and alias in main.relay[relay]["users"]: + # return "PROVISIONED" + #else: + # main.relay[relay]["networks"] = [] + #main.relay[relay]["networks"].append(network) provisionNetworkData(relay, alias, network, main.network[network]["host"], main.network[network]["port"], main.network[network]["security"], main.network[network]["auth"], main.alias[alias]["password"]) - main.saveConf("relay") + #main.saveConf("relay") storedNetwork = False num = 1 while not storedNetwork: i = str(num) if num == 1000: - error("Too many iterations in while trying to choose name for r: %s a: %s n: %s" % (relay, alias, network)) + error("Iteration limit exceeded while trying to choose name for r: %s a: %s n: %s" % (relay, alias, network)) return False if network+i in main.pool.keys(): From f34ddab6fc5b8eeecc47b4dd477926a9bc05447c Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 15 Aug 2019 21:20:49 +0100 Subject: [PATCH 3/6] Improvements to query and self event detection, implement all command and debug flags --- commands/all.py | 23 ++++++++++++++++++++++ conf/example/config.json | 2 ++ conf/help.json | 3 ++- core/bot.py | 35 +++++++++++++++++++-------------- main.py | 6 ++++++ modules/monitor.py | 12 ++++++++--- modules/provision.py | 3 ++- modules/userinfo.py | 7 +++++++ threshold | 4 +++- utils/dedup.py | 21 ++++++++++++++++++++ utils/loaders/command_loader.py | 2 +- utils/logging/debug.py | 7 +++++++ utils/logging/log.py | 3 --- 13 files changed, 103 insertions(+), 25 deletions(-) create mode 100644 commands/all.py create mode 100644 utils/dedup.py create mode 100644 utils/logging/debug.py diff --git a/commands/all.py b/commands/all.py new file mode 100644 index 0000000..10ec381 --- /dev/null +++ b/commands/all.py @@ -0,0 +1,23 @@ +import main +from core.bot import deliverRelayCommands + +class All: + def __init__(self, *args): + self.all(*args) + + def all(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + if authed: + if length > 2: + for i in main.pool.keys(): + relay = main.pool[i]["relay"] + network = main.pool[i]["network"] + alias = main.pool[i]["alias"] + commands = {spl[1]: [" ".join(spl[2:])]} + success("Sending commands to relay %s as user %s" % (relay, alias+"/"+network)) + deliverRelayCommands(relay, commands, user=alias+"/"+network) + return + else: + incUsage("all") + return + else: + incUsage(None) diff --git a/conf/example/config.json b/conf/example/config.json index bc67bdf..5ecb53f 100644 --- a/conf/example/config.json +++ b/conf/example/config.json @@ -15,6 +15,7 @@ "RedisSocket": "/tmp/redis.sock", "UsePassword": true, "ConnectOnCreate": false, + "Debug": false, "Dist": { "Enabled": true, "SendOutput": false, @@ -26,6 +27,7 @@ }, "Password": "s", "Tweaks": { + "MaxHash": 10, "ZNC": { "Prefix": "*" }, diff --git a/conf/help.json b/conf/help.json index 0211171..522f00e 100644 --- a/conf/help.json +++ b/conf/help.json @@ -24,5 +24,6 @@ "network": "network [
]", "provision": "provision []", "cmd": "cmd ", - "token": "token [] []" + "token": "token [] []", + "all": "all " } diff --git a/core/bot.py b/core/bot.py index ae4a0fe..33a20e9 100644 --- a/core/bot.py +++ b/core/bot.py @@ -9,10 +9,11 @@ from random import randint from copy import deepcopy from modules import userinfo -from modules import counters as count +from modules import counters from modules import monitor from core.relay import sendRelayNotification +from utils.dedup import dedup import main from utils.logging.log import * @@ -151,9 +152,9 @@ class IRCBot(IRCClient): del cast["host"] del cast["target"] if not cast["type"] in ["query", "self", "highlight", "znc", "who"]: - if "target" in cast.keys(): - if cast["target"].lower() == self.nickname.lower(): - #castDup = deepcopy(cast) + if "target" in cast.keys() and not cast["type"] == "mode": # don't handle modes here + if cast["target"].lower() == self.nickname.lower(): # as they are target == nickname + #castDup = deepcopy(cast) # however modes are not queries! cast["mtype"] = cast["type"] cast["type"] = "query" cast["name"] = self.name @@ -173,7 +174,8 @@ class IRCBot(IRCClient): castDup["mtype"] = cast["type"] castDup["type"] = "self" castDup["name"] = self.name - self.event(**castDup) + if not cast["target"].lower() == self.nickname.lower(): # modes has been set on us directly + self.event(**castDup) # don't tell anyone else if "message" in cast.keys() and not cast["type"] == "query": # Don't highlight queries if self.nickname.lower() in cast["message"].lower(): castDup = deepcopy(cast) @@ -184,8 +186,8 @@ class IRCBot(IRCClient): if not "name" in cast.keys(): cast["name"] = self.net - count.event(self.net, cast["type"]) - monitor.event(cast) + counters.event(self.net, cast["type"]) + monitor.event(self.name, cast) def privmsg(self, user, channel, msg): self.event(type="msg", muser=user, target=channel, message=msg) @@ -408,37 +410,40 @@ class IRCBot(IRCClient): self.event(type="part", muser=user, target=channel, message=message) def userQuit(self, user, quitMessage): - self.chanlessEvent(type="quit", muser=user, message=quitMessage) + self.chanlessEvent({"type": "quit", "muser": user, "message": quitMessage}) def userKicked(self, kickee, channel, kicker, message): if kickee.lower() == self.nickname.lower(): self.botLeft(channel) self.event(type="kick", muser=kicker, target=channel, message=message, user=kickee) - def chanlessEvent(self, **cast): - chans = userinfo.getChansSingle(self.net, cast["nick"]) + def chanlessEvent(self, cast): + cast["nick"], cast["ident"], cast["host"] = self.parsen(cast["muser"]) + if dedup(self.name, cast): + return # stop right there sir! + chans = userinfo.getChanList(self.net, cast["nick"]) if chans == None: - self.event(**cast) + error("No channels returned for chanless event: %s" % cast) + # self.event(**cast) -- no, should NEVER happen return # getChansSingle returns all channels of the user, we only want to use # ones we have common with them realChans = set(chans).intersection(set(self.channels)) - for i in chans: + for i in realChans: cast["target"] = i self.event(**cast) def userRenamed(self, oldname, newname): - self.chanlessEvent(type="nick", muser=oldname, user=newname) + self.chanlessEvent({"type": "nick", "muser": oldname, "user": newname}) def topicUpdated(self, user, channel, newTopic): self.event(type="topic", muser=user, target=channel, message= newTopic) def modeChanged(self, user, channel, toset, modes, args): - nick, ident, host = self.parsen(user) argList = list(args) modeList = [i for i in modes] for a, m in zip(argList, modeList): - self.event(type="mode", nick=nick, ident=ident, host=host, target=channel, modes=m, status=toset, modeargs=a) + self.event(type="mode", muser=user, target=channel, modes=m, status=toset, modeargs=a) class IRCBotFactory(ReconnectingClientFactory): def __init__(self, name, relay=None, relayCommands=None, user=None, stage2=None): diff --git a/main.py b/main.py index 1aa0cdf..ccb6942 100644 --- a/main.py +++ b/main.py @@ -1,6 +1,8 @@ from json import load, dump, loads from redis import StrictRedis from string import digits +from os import urandom + from utils.logging.log import * configPath = "conf/" @@ -31,6 +33,10 @@ CommandMap = {} runningSample = 0 lastMinuteSample = 0 +# Generate 16-byte hex key for message checksums +hashKey = urandom(16) +lastEvents = {} + def nets(): if not "pool" in globals(): return diff --git a/modules/monitor.py b/modules/monitor.py index 16b0bb4..3913c6f 100644 --- a/modules/monitor.py +++ b/modules/monitor.py @@ -1,9 +1,11 @@ from copy import deepcopy from json import dumps +from datetime import datetime import main from core.relay import sendRelayNotification from modules import userinfo +from utils.dedup import dedup def testNetTarget(name, target): called = False @@ -51,10 +53,12 @@ def magicFunction(A, B): else: return all(A[k] in B[k] for k in set(A) & set(B)) and set(B) <= set(A) -def event(c): # yes I'm using a short variable because otherwise it goes off the screen +def event(numName, c): # yes I'm using a short variable because otherwise it goes off the screen if not "target" in c.keys(): c["target"] = None + if dedup(numName, c): + return # metadata scraping # need to check if this was received from a relay # in which case, do not do this @@ -68,9 +72,9 @@ def event(c): # yes I'm using a short variable because otherwise it goes off the elif c["type"] == "quit": userinfo.delUserByNetwork(c["name"], c["nick"], c["muser"]) elif c["type"] == "join": - userinfo.addUser(c["name"], c["target"], c["nick"], c["user"]) + userinfo.addUser(c["name"], c["target"], c["nick"], c["muser"]) elif c["type"] == "part": - userinfo.delUser(c["name"], c["target"], c["nick"], c["user"]) + userinfo.delUser(c["name"], c["target"], c["nick"], c["muser"]) if "mtype" in c.keys(): if c["mtype"] == "nick": @@ -79,6 +83,8 @@ def event(c): # yes I'm using a short variable because otherwise it goes off the if "muser" in c.keys(): del c["muser"] sendRelayNotification(c) + + # only monitors below monitorGroups = testNetTarget(c["name"], c["target"]) if monitorGroups == False: return diff --git a/modules/provision.py b/modules/provision.py index f964cf3..fb3ea9c 100644 --- a/modules/provision.py +++ b/modules/provision.py @@ -34,7 +34,7 @@ def provisionNetworkData(relay, alias, network, host, port, security, auth, pass elif auth == "ns": stage2commands["status"] = [] stage2commands["nickserv"] = [] - stage2commands["status"].append("LoadMod NickServ") + stage2commands["status"].append("LoadMod nickserv") stage2commands["nickserv"].append("Set %s" % password) if not main.config["ConnectOnCreate"]: stage3commands["status"] = [] @@ -42,6 +42,7 @@ def provisionNetworkData(relay, alias, network, host, port, security, auth, pass if main.config["Toggles"]["CycleChans"]: stage2commands["status"] = [] stage2commands["status"].append("LoadMod disconkick") + stage2commands["status"].append("LoadMod chansaver") deliverRelayCommands(relay, commands, stage2=[[alias+"/"+network, stage2commands], [alias+"/"+network, stage3commands]]) diff --git a/modules/userinfo.py b/modules/userinfo.py index d9bdddf..00020cd 100644 --- a/modules/userinfo.py +++ b/modules/userinfo.py @@ -23,6 +23,13 @@ def getChansSingle(name, nick): return None return [i.decode() for i in result] +def getChanList(name, nick): + chanspace = "live.chan."+name+"."+nick + result = main.r.smembers(chanspace) + if len(result) == 0: + return None + return [i.decode() for i in result] + def getChans(nick): result = {} for i in main.nets(): diff --git a/threshold b/threshold index 5ff3a3f..648ed4f 100755 --- a/threshold +++ b/threshold @@ -1,6 +1,7 @@ #!/usr/bin/env python from twisted.internet import reactor from twisted.internet.ssl import DefaultOpenSSLContextFactory +from sys import argv #from twisted.python import log #from sys import stdout #log.startLogging(stdout) @@ -8,7 +9,8 @@ from twisted.internet.ssl import DefaultOpenSSLContextFactory import main main.initMain() - +if "--debug" in argv: # yes really + main.config["Debug"] = True from utils.logging.log import * from utils.loaders.command_loader import loadCommands from core.helper import startBot diff --git a/utils/dedup.py b/utils/dedup.py new file mode 100644 index 0000000..ee07bb5 --- /dev/null +++ b/utils/dedup.py @@ -0,0 +1,21 @@ +from datetime import datetime +from csiphash import siphash24 +from json import dumps +import main +from utils.logging.debug import debug + +def dedup(numName, c): + # deduplication + c["approxtime"] = int(datetime.utcnow().timestamp()) + castHash = siphash24(main.hashKey, dumps(c, sort_keys=True).encode("utf-8")) + del c["approxtime"] + isDuplicate= any(castHash in main.lastEvents[x] for x in main.lastEvents.keys() if not x == numName) + if isDuplicate: + debug("Duplicate: %s" % (c)) + return True + if numName in main.lastEvents.keys(): + main.lastEvents[numName].insert(0, castHash) + main.lastEvents[numName] = main.lastEvents[numName][0:main.config["Tweaks"]["MaxHash"]] + else: + main.lastEvents[numName] = [castHash] + return False diff --git a/utils/loaders/command_loader.py b/utils/loaders/command_loader.py index e3ea253..ad623cc 100644 --- a/utils/loaders/command_loader.py +++ b/utils/loaders/command_loader.py @@ -1,6 +1,6 @@ from os import listdir -from utils.logging.log import * +from utils.logging.debug import debug import commands from main import CommandMap diff --git a/utils/logging/debug.py b/utils/logging/debug.py new file mode 100644 index 0000000..5a7b067 --- /dev/null +++ b/utils/logging/debug.py @@ -0,0 +1,7 @@ +import main +# we need a seperate module to log.py, as log.py is imported by main.py, and we need to access main +# to read the setting +def debug(data): + if main.config["Debug"]: + print("[DEBUG]", data) + diff --git a/utils/logging/log.py b/utils/logging/log.py index 5e1141f..6bb6a99 100644 --- a/utils/logging/log.py +++ b/utils/logging/log.py @@ -1,9 +1,6 @@ def log(data): print("[LOG]", data) -def debug(data): - print("[DEBUG]", data) - def warn(data): print("[WARNING]", data) From 22bd0d3ac61107eb2a2f8674f2785b9c74b8eba1 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 15 Aug 2019 22:14:45 +0100 Subject: [PATCH 4/6] Don't crash if the part message is null --- core/bot.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/core/bot.py b/core/bot.py index 33a20e9..09f3ce7 100644 --- a/core/bot.py +++ b/core/bot.py @@ -175,14 +175,15 @@ class IRCBot(IRCClient): castDup["type"] = "self" castDup["name"] = self.name if not cast["target"].lower() == self.nickname.lower(): # modes has been set on us directly - self.event(**castDup) # don't tell anyone else + self.event(**castDup) # don't tell anyone else if "message" in cast.keys() and not cast["type"] == "query": # Don't highlight queries - if self.nickname.lower() in cast["message"].lower(): - castDup = deepcopy(cast) - castDup["mtype"] = cast["type"] - castDup["type"] = "highlight" - castDup["name"] = self.name - self.event(**castDup) + if not cast["message"] == None: + if self.nickname.lower() in cast["message"].lower(): + castDup = deepcopy(cast) + castDup["mtype"] = cast["type"] + castDup["type"] = "highlight" + castDup["name"] = self.name + self.event(**castDup) if not "name" in cast.keys(): cast["name"] = self.net @@ -380,8 +381,8 @@ class IRCBot(IRCClient): if not channel in self.channels: self.channels.append(channel) self.names(channel).addCallback(self.got_names) - self.who(channel).addCallback(self.got_who) if main.config["Toggles"]["Who"]: + self.who(channel).addCallback(self.got_who) lc = LoopingCall(self.who, channel) self._getWho[channel] = lc intrange = main.config["Tweaks"]["Delays"]["WhoRange"] From 545282e201906b0e33f94a2e28c9a65a61df21fc Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Fri, 16 Aug 2019 21:27:23 +0100 Subject: [PATCH 5/6] Add deduplication precision toggle, fix printing odd characters and implement sending messages to all instances of a certain network, or all networks associated with a certain alias --- commands/allc.py | 46 ++++++++++++++++++++++++++++++++++++++++ conf/example/config.json | 1 + conf/help.json | 3 ++- threshold | 7 ++++-- utils/dedup.py | 2 +- 5 files changed, 55 insertions(+), 4 deletions(-) create mode 100644 commands/allc.py diff --git a/commands/allc.py b/commands/allc.py new file mode 100644 index 0000000..9e662fe --- /dev/null +++ b/commands/allc.py @@ -0,0 +1,46 @@ +import main +from core.bot import deliverRelayCommands + +class Allc: + def __init__(self, *args): + self.allc(*args) + + def allc(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): + if authed: + if length > 4: + targets = [] + if spl[1] == "network": + if spl[2] in main.network.keys(): + for i in main.pool.keys(): + if main.pool[i]["network"] == spl[2]: + targets.append(i) + else: + failure("No such network: %s" % spl[2]) + return + elif spl[1] == "alias": + if spl[2] in main.alias.keys(): + for i in main.pool.keys(): + if main.pool[i]["alias"] == spl[2]: + targets.append(i) + else: + failure("No such alias: %s" % spl[2]) + return + else: + incUsage("allc") + return + if len(targets) == 0: + failure("No matches found: %s" % spl[2]) + return + for i in targets: + relay = main.pool[i]["relay"] + network = main.pool[i]["network"] + alias = main.pool[i]["alias"] + commands = {spl[3]: [" ".join(spl[4:])]} + success("Sending commands to relay %s as user %s" % (relay, alias+"/"+network)) + deliverRelayCommands(relay, commands, user=alias+"/"+network) + return + else: + incUsage("allc") + return + else: + incUsage(None) diff --git a/conf/example/config.json b/conf/example/config.json index 5ecb53f..a595963 100644 --- a/conf/example/config.json +++ b/conf/example/config.json @@ -28,6 +28,7 @@ "Password": "s", "Tweaks": { "MaxHash": 10, + "DedupPrecision": 9, "ZNC": { "Prefix": "*" }, diff --git a/conf/help.json b/conf/help.json index 522f00e..1e55f3a 100644 --- a/conf/help.json +++ b/conf/help.json @@ -25,5 +25,6 @@ "provision": "provision []", "cmd": "cmd ", "token": "token [] []", - "all": "all " + "all": "all ", + "allc": "allc <(network)|(alias)> " } diff --git a/threshold b/threshold index 648ed4f..d63a8b2 100755 --- a/threshold +++ b/threshold @@ -1,11 +1,14 @@ #!/usr/bin/env python from twisted.internet import reactor from twisted.internet.ssl import DefaultOpenSSLContextFactory -from sys import argv +from sys import argv, stdout, stderr #from twisted.python import log #from sys import stdout #log.startLogging(stdout) - +from codecs import getwriter # fix printing odd shit to the terminal +stdout = getwriter("utf8")(stdout) # this is a generic fix but we all know +stderr = getwriter("utf8")(stderr) # it's just for the retards on Rizon using + # unicode quit messages for no reason import main main.initMain() diff --git a/utils/dedup.py b/utils/dedup.py index ee07bb5..03255e3 100644 --- a/utils/dedup.py +++ b/utils/dedup.py @@ -6,7 +6,7 @@ from utils.logging.debug import debug def dedup(numName, c): # deduplication - c["approxtime"] = int(datetime.utcnow().timestamp()) + c["approxtime"] = str(datetime.utcnow().timestamp())[:main.config["Tweaks"]["DedupPrecision"]] castHash = siphash24(main.hashKey, dumps(c, sort_keys=True).encode("utf-8")) del c["approxtime"] isDuplicate= any(castHash in main.lastEvents[x] for x in main.lastEvents.keys() if not x == numName) From c63f301b7f785971b121c12aa040535169c60659 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Mon, 19 Aug 2019 20:12:42 +0100 Subject: [PATCH 6/6] Defer initialUsers, initialNames and delChannel to threads to improve performance --- core/bot.py | 2 +- modules/userinfo.py | 31 +++++++++++++++++++++++++------ 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/core/bot.py b/core/bot.py index 09f3ce7..0f41dfe 100644 --- a/core/bot.py +++ b/core/bot.py @@ -398,7 +398,7 @@ class IRCBot(IRCClient): lc.stop() del self._getWho[channel] userinfo.delChannel(self.net, channel) # < we do not need to deduplicate this - log("Can no longer cover %s, removing records" % channel) # as it will only be matched once -- + #log("Can no longer cover %s, removing records" % channel)# as it will only be matched once -- # other bots have different nicknames so def left(self, user, channel, message): # even if they saw it, they wouldn't react self.event(type="part", muser=user, target=channel, message=message) diff --git a/modules/userinfo.py b/modules/userinfo.py index 00020cd..29da9b0 100644 --- a/modules/userinfo.py +++ b/modules/userinfo.py @@ -1,6 +1,9 @@ -import main +from twisted.internet.threads import deferToThread from string import digits + +import main from utils.logging.log import * +from utils.logging.debug import debug def getWhoSingle(name, query): result = main.r.sscan("live.who."+name, 0, query, count=9999999) @@ -68,14 +71,19 @@ def getNamespace(name, channel, nick): chanspace = "live.chan.%s.%s" % (name, nick) return [gnamespace, namespace, chanspace] -def initialUsers(name, channel, users): +def _initialUsers(name, channel, users): gnamespace = "live.who.%s" % name p = main.r.pipeline() for i in users: p.sadd(gnamespace, i[0]+"!"+i[1]+"@"+i[2]) p.execute() -def initialNames(name, channel, names): +def initialUsers(name, channel, users): + debug("Initialising WHO records for %s on %s" % (channel, name)) + d = deferToThread(_initialUsers, name, channel, users) + #d.addCallback(testCallback) + +def _initialNames(name, channel, names): namespace = "live.who.%s.%s" % (name, channel) p = main.r.pipeline() for i in names: @@ -83,6 +91,11 @@ def initialNames(name, channel, names): p.sadd("live.chan."+name+"."+i, channel) p.execute() +def initialNames(name, channel, names): + debug("Initialising NAMES records for %s on %s" % (channel, name)) + d = deferToThread(_initialNames, name, channel, names) + #d.addCallback(testCallback) + def editUser(name, user): gnamespace = "live.who.%s" % name main.r.sadd(gnamespace, user) @@ -158,7 +171,7 @@ def delUserByNetwork(name, nick, user): p.delete(chanspace) p.execute() -def delChannel(name, channel): # This function is extremely expensive, look to replace +def _delChannel(name, channel): # This function is extremely expensive, look to replace gnamespace = "live.who.%s" % name namespace = "live.who.%s.%s" % (name, channel) p = main.r.pipeline() @@ -173,10 +186,16 @@ def delChannel(name, channel): # This function is extremely expensive, look to r p.srem("live.chan."+name+"."+i.decode(), channel) p.delete(namespace) p.execute() + return [name, channel] + +def delChannel(name, channel): + debug("Purging channel %s for %s" % (channel, name)) + d = deferToThread(_delChannel, name, channel) + #d.addCallback(testCallback) def delNetwork(name, channels): - log("Purging channels for %s" % name) + debug("Purging channels for %s" % name) for i in channels: delChannel(name, i) - log("Finished purging channels for %s" % name) + #log("Finished purging channels for %s" % name) return