From 36105c7e9a78d4a1832463c5eeb13d4612e0eb3f Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Mon, 27 Aug 2018 20:42:49 +0100 Subject: [PATCH] Move user metadata info into redis --- commands/stats.py | 15 +---- commands/who.py | 4 +- conf/help.json | 4 +- core/bot.py | 107 +++++++++++++++++++++++--------- main.py | 6 +- modules/keyword.py | 1 - modules/userinfo.py | 146 +++++++++++++++++++++++++++++++++++++------- 7 files changed, 213 insertions(+), 70 deletions(-) diff --git a/commands/stats.py b/commands/stats.py index 6d032f3..35d2d40 100644 --- a/commands/stats.py +++ b/commands/stats.py @@ -1,5 +1,6 @@ import main import modules.counters as count +import modules.userinfo as userinfo class Stats: def __init__(self, register): @@ -13,8 +14,7 @@ class Stats: numWhoEntries = 0 for i in main.IRCPool.keys(): numChannels += len(main.IRCPool[i].channels) - for i in main.wholist.keys(): - numWhoEntries += len(main.wholist[i].keys()) + numWhoEntries += userinfo.getNumTotalWhoEntries() stats.append("Servers: %s" % len(main.pool.keys())) stats.append("Online servers: %s" % len(main.IRCPool.keys())) stats.append("Channels: %s" % numChannels) @@ -34,20 +34,11 @@ class Stats: numChannels = 0 numWhoEntries = 0 - failures = 0 if spl[1] in main.IRCPool.keys(): numChannels += len(main.IRCPool[spl[1]].channels) else: failure("Name does not exist: %s" % spl[1]) - failures += 1 - if spl[1] in main.wholist.keys(): - numWhoEntries += len(main.wholist[spl[1]].keys()) - else: - failure("Who entry does not exist: %s" % spl[1]) - failures += 1 - if failures == 2: - failure("No information found, aborting") - return + numWhoEntries += userinfo.getNumWhoEntries(spl[1]) stats.append("Channels: %s" % numChannels) stats.append("User records: %s" % numWhoEntries) counterEvents = count.getEvents(spl[1]) diff --git a/commands/who.py b/commands/who.py index a2b4cfb..9898cfc 100644 --- a/commands/who.py +++ b/commands/who.py @@ -14,9 +14,9 @@ class Who: rtrn += "Matches from: %s" % i rtrn += "\n" for x in result[i]: - x = [y for y in x if not y == None] - rtrn += str((", ".join(x))) + rtrn += (x) rtrn += "\n" + rtrn += "\n" info(rtrn) return else: diff --git a/conf/help.json b/conf/help.json index a69df80..e2bed17 100644 --- a/conf/help.json +++ b/conf/help.json @@ -14,8 +14,8 @@ "disable": "disable ", "list": "list", "stats": "stats []", - "save": "save ", - "load": "load ", + "save": "save ", + "load": "load ", "dist": "dist", "loadmod": "loadmod ", "msg": "msg ", diff --git a/core/bot.py b/core/bot.py index 04e881d..98ecf31 100644 --- a/core/bot.py +++ b/core/bot.py @@ -2,6 +2,7 @@ from twisted.internet.protocol import ReconnectingClientFactory from twisted.words.protocols.irc import IRCClient from twisted.internet.defer import Deferred from twisted.internet.task import LoopingCall + from string import digits from random import randint @@ -19,6 +20,8 @@ class IRCBot(IRCClient): self.connected = False self.channels = [] self.net = "".join([x for x in name if not x in digits]) + if self.net == "": + error("Network with all numbers: %s" % name) self.buffer = "" self.name = name instance = main.pool[name] @@ -37,6 +40,8 @@ class IRCBot(IRCClient): self._who = {} self._getWho = {} + self._names = {} + self.authtype = instance["authtype"] if self.authtype == "ns": self.authpass = instance["password"] @@ -71,7 +76,7 @@ class IRCBot(IRCClient): def privmsg(self, user, channel, msg): nick, ident, host = self.parsen(user) - userinfo.setWhoSingle(self.net, nick, ident, host) + userinfo.editUser(self.net, channel, nick, user) count.event(self.name, "privmsg") keyword.actKeyword(user, channel, msg, self.nickname, "MSG", self.name) @@ -79,7 +84,7 @@ class IRCBot(IRCClient): def noticed(self, user, channel, msg): nick, ident, host = self.parsen(user) - userinfo.setWhoSingle(self.net, nick, ident, host) + userinfo.editUser(self.net, channel, nick, user) count.event(self.name, "notice") keyword.actKeyword(user, channel, msg, self.nickname, "NOTICE", self.name) @@ -87,7 +92,7 @@ class IRCBot(IRCClient): def action(self, user, channel, msg): nick, ident, host = self.parsen(user) - userinfo.setWhoSingle(self.net, nick, ident, host) + userinfo.editUser(self.net, channel, nick, user) count.event(self.name, "action") keyword.actKeyword(user, channel, msg, self.nickname, "ACTION", self.name) @@ -109,8 +114,10 @@ class IRCBot(IRCClient): newnick = nickname + "_" return newnick - def nickChanged(self, nick): - self.nickname = nick + def nickChanged(self, olduser, newnick): + oldnick, ident, host = self.parsen(olduser) + userinfo.renameUser(self.net, oldnick, olduser, newnick, newnick+"!"+ident+"@"+host) + self.nickname = newnick count.event(self.name, "selfnick") def irc_ERR_NICKNAMEINUSE(self, prefix, params): @@ -122,10 +129,9 @@ class IRCBot(IRCClient): sendAll("%s: password mismatch" % self.name) def who(self, channel): - channel = channel d = Deferred() if channel not in self._who: - self._who[channel] = ([], {}) + self._who[channel] = ([], []) self._who[channel][0].append(d) self.sendLine("WHO %s" % channel) return d @@ -141,7 +147,7 @@ class IRCBot(IRCClient): if channel not in self._who: return n = self._who[channel][1] - n[nick] = [nick, user, host, server, status, realname] + n.append([nick, user, host, server, status, realname]) count.event(self.name, "whoreply") monitor.event(self.net, channel, {"type": "who", "exact": nick+"!"+user+"@"+host, "nick": nick, "ident": user, "host": host, "realname": realname, "server": server, "status": status}) @@ -155,7 +161,50 @@ class IRCBot(IRCClient): del self._who[channel] def got_who(self, whoinfo): - userinfo.setWho(self.name, whoinfo[1]) + userinfo.initialUsers(self.net, whoinfo[0], whoinfo[1]) + + def sanit(self, data): + if len(data) >= 1: + if data[0] in ["!", "~", "&", "@", "%", "+"]: + data = data[1:] + return data + return data + else: + return False + + def names(self, channel): + d = Deferred() + if channel not in self._names: + self._names[channel] = ([], []) + self._names[channel][0].append(d) + self.sendLine("NAMES %s" % channel) + return d + + def irc_RPL_NAMREPLY(self, prefix, params): + channel = params[2] + nicklist = params[3].split(' ') + if channel not in self._names: + return + n = self._names[channel][1] + n.append(nicklist) + + def irc_RPL_ENDOFNAMES(self, prefix, params): + channel = params[1] + if channel not in self._names: + return + callbacks, namelist = self._names[channel] + for cb in callbacks: + cb.callback((channel, namelist)) + del self._names[channel] + + def got_names(self, nicklist): + newNicklist = [] + for i in nicklist[1]: + for x in i: + f = self.sanit(x) + if f: + newNicklist.append(f) + userinfo.initialNames(self.net, nicklist[0], newNicklist) #twisted sucks so i have to do this to actually get the user info def irc_JOIN(self, prefix, params): @@ -197,7 +246,7 @@ class IRCBot(IRCClient): """ nick = prefix.split('!', 1)[0] if nick == self.nickname: - self.nickChanged(params[0]) + self.nickChanged(prefix, params[0]) else: self.userRenamed(prefix, params[0]) @@ -236,61 +285,63 @@ class IRCBot(IRCClient): self.join(i) count.event(self.name, "signedon") - def getWho(self, channel): - self.who(channel).addCallback(self.got_who) - def joined(self, channel): if not channel in self.channels: self.channels.append(channel) + self.names(channel).addCallback(self.got_names) self.who(channel).addCallback(self.got_who) count.event(self.name, "selfjoin") if self.name == main.config["Master"][0] and channel == main.config["Master"][1]: for i in range(len(main.masterbuf)): self.msg(channel, main.masterbuf.pop(0)) main.saveConf("masterbuf") - lc = LoopingCall(self.getWho, channel) + lc = LoopingCall(self.who, channel) self._getWho[channel] = lc intrange = main.config["Tweaks"]["Delays"]["WhoRange"] minint = main.config["Tweaks"]["Delays"]["WhoLoop"] interval = randint(minint, minint+intrange) lc.start(interval) - def left(self, channel, message): + def botLeft(self, channel): if channel in self.channels: self.channels.remove(channel) + if channel in self._getWho.keys(): + lc = self._getWho[channel] + lc.stop() + del self._getWho[channel] + userinfo.delChannel(self.net, channel) + + def left(self, channel, message): keyword.actKeyword(self.nickname, channel, message, self.nickname, "SELFPART", self.name) count.event(self.name, "selfpart") monitor.event(self.net, channel, {"type": "part", "message": message}) - lc = self._getWho[channel] - lc.stop() + self.botLeft(channel) def kickedFrom(self, channel, kicker, message): nick, ident, host = self.parsen(kicker) - userinfo.setWhoSingle(self.net, nick, ident, host) if channel in self.channels: self.channels.remove(channel) keyword.sendMaster("KICK %s: (%s/%s) %s" % (self.name, kicker, channel, message)) count.event(self.name, "selfkick") monitor.event(self.net, channel, {"type": "kick", "exact": kicker, "nick": nick, "ident": ident, "host": host, "message": message}) - lc = self._getWho[channel] - lc.stop() + self.botLeft(channel) def userJoined(self, user, channel): nick, ident, host = self.parsen(user) - userinfo.setWhoSingle(self.net, nick, ident, host) + userinfo.addUser(self.net, channel, nick, user) count.event(self.name, "join") monitor.event(self.net, channel, {"type": "join", "exact": user, "nick": nick, "ident": ident, "host": host}) def userLeft(self, user, channel, message): nick, ident, host = self.parsen(user) - userinfo.setWhoSingle(self.net, nick, ident, host) + userinfo.delUser(self.net, channel, nick, user) keyword.actKeyword(user, channel, message, self.nickname, "PART", self.name) count.event(self.name, "part") monitor.event(self.net, channel, {"type": "part", "exact": user, "nick": nick, "ident": ident, "host": host, "message": message}) def userQuit(self, user, quitMessage): nick, ident, host = self.parsen(user) - userinfo.setWhoSingle(self.net, nick, ident, host) + userinfo.delUserByNetwork(self.net, nick, user) count.event(self.name, "quit") keyword.actKeyword(user, None, quitMessage, self.nickname, "QUIT", self.name) @@ -298,7 +349,8 @@ class IRCBot(IRCClient): def userKicked(self, kickee, channel, kicker, message): nick, ident, host = self.parsen(kicker) - userinfo.setWhoSingle(self.net, nick, ident, host) + userinfo.editUser(self.net, channel, nick, kicker) + userinfo.delUserByNick(self.net, channel, kickee) count.event(self.name, "kick") keyword.actKeyword(kicker, channel, message, self.nickname, "KICK", self.name) @@ -306,14 +358,13 @@ class IRCBot(IRCClient): def userRenamed(self, oldname, newname): nick, ident, host = self.parsen(oldname) - userinfo.setWhoSingle(self.net, nick, ident, host) - userinfo.setWhoSingle(self.net, newname, ident, host) + userinfo.renameUser(self.net, nick, oldname, newname, newname+"!"+ident+"@"+host) count.event(self.name, "nick") monitor.event(self.net, None, {"type": "nick", "exact": oldname, "nick": nick, "ident": ident, "host": host, "user": newname}) def topicUpdated(self, user, channel, newTopic): nick, ident, host = self.parsen(user) - userinfo.setWhoSingle(self.net, nick, ident, host) + userinfo.editUser(self.net, channel, nick, user) count.event(self.name, "topic") keyword.actKeyword(user, channel, newTopic, self.nickname, "TOPIC", self.name) @@ -321,7 +372,7 @@ class IRCBot(IRCClient): def modeChanged(self, user, channel, toset, modes, args): nick, ident, host = self.parsen(user) - userinfo.setWhoSingle(self.net, nick, ident, host) + userinfo.editUser(self.net, channel, nick, user) count.event(self.name, "mode") argList = list(args) modeList = [i for i in modes] diff --git a/main.py b/main.py index 6423266..985bbea 100644 --- a/main.py +++ b/main.py @@ -1,8 +1,10 @@ from json import load, dump, loads - +import redis from utils.loaders.command_loader import loadCommands from utils.logging.log import * +r = redis.StrictRedis(unix_socket_path='/tmp/redis.sock', db=0) + configPath = "conf/" certPath = "cert/" @@ -11,7 +13,7 @@ filemap = { "keyconf": ["keyword.json", "keyword lists"], "pool": ["pool.json", "pool"], "help": ["help.json", "command help"], - "wholist": ["wholist.json", "WHO lists"], +# "wholist": ["wholist.json", "WHO lists"], "counters": ["counters.json", "counters file"], "masterbuf": ["masterbuf.json", "master buffer"], "monitor": ["monitor.json", "monitoring database"], diff --git a/modules/keyword.py b/modules/keyword.py index 40638a4..66a588b 100644 --- a/modules/keyword.py +++ b/modules/keyword.py @@ -29,7 +29,6 @@ def sendMaster(data): if not hasMonitors: main.masterbuf.append(data) main.saveConf("masterbuf") - warn("Master with no IRC instance defined") for i in main.MonitorPool: main.connections[i].send(data) diff --git a/modules/userinfo.py b/modules/userinfo.py index 952075d..ebb9dda 100644 --- a/modules/userinfo.py +++ b/modules/userinfo.py @@ -1,31 +1,131 @@ import main from string import digits -#from utils.logging.log import * +from utils.logging.log import * -def setWho(network, newObjects): - if not network in main.wholist.keys(): - main.wholist[network] = {} - for i in newObjects.keys(): - main.wholist[network][i] = newObjects[i] +def getWhoSingle(name, query): + result = main.r.sscan("live.who."+name, 0, query) + if result[1] == []: + return None + return [i.decode() for i in result[1]] - return +def getWho(query): + result = {} + for i in main.pool.keys(): + f = getWhoSingle(i, query) + if f: + result[i] = f + print(result) + return result + +def getNumWhoEntries(name): + return main.r.scard("live.who."+name) + +def getNumTotalWhoEntries(): + total = 0 + for i in main.pool.keys(): + total += getNumWhoEntries(i) + return total + +def getNamespace(name, channel, nick): + gnamespace = "live.who.%s" % name + namespace = "live.who.%s.%s" % (name, channel) + chanspace = "live.chan.%s.%s" % (name, nick) + return [gnamespace, namespace, chanspace] + +def initialUsers(name, channel, users): + gnamespace = "live.who.%s" % name + p = main.r.pipeline() + for i in users: + p.sadd(gnamespace, i[0]+"!"+i[1]+"@"+i[2]) + p.execute() + +def initialNames(name, channel, names): + namespace = "live.who.%s.%s" % (name, channel) + p = main.r.pipeline() + for i in names: + p.sadd(namespace, i) + p.sadd("live.chan."+name+"."+i, channel) + p.execute() + +def editUser(name, channel, nick, user): + gnamespace = "live.who.%s" % name + main.r.sadd(gnamespace, user) + +def addUser(name, channel, nick, user): + gnamespace, namespace, chanspace = getNamespace(name, channel, nick) + p = main.r.pipeline() + p.sadd(gnamespace, user) + p.sadd(namespace, nick) + p.sadd(chanspace, channel) + p.execute() -def setWhoSingle(network, nick, ident, host): - if network in main.wholist.keys(): - if nick in main.wholist[network].keys(): - main.wholist[network][nick][1] = ident - main.wholist[network][nick][2] = host +def delUser(name, channel, nick, user): + gnamespace, namespace, chanspace = getNamespace(name, channel, nick) + p = main.r.pipeline() + channels = main.r.smembers(chanspace) + p.srem(namespace, nick) + if channels == {channel.encode()}: + p.delete(chanspace) + p.srem(gnamespace, user) + else: + p.srem(chanspace, channel) + p.execute() + +def getUserByNick(name, nick): + gnamespace = "live.who.%s" % name + usermatch = main.r.sscan(gnamespace, match=nick+"!*") + if usermatch[1] == []: + return False + else: + if len(usermatch[1]) == 1: + user = usermatch[1][0] + return user else: - main.wholist[network][nick] = [nick, ident, host, None, None, None] + return False + +def renameUser(name, oldnick, olduser, newnick, newuser): + gnamespace = "live.who.%s" % name + chanspace = "live.chan.%s.%s" % (name, oldnick) + newchanspace = "live.chan.%s.%s" % (name, newnick) + p = main.r.pipeline() + p.srem(gnamespace, olduser) + p.sadd(gnamespace, newuser) + for i in main.r.smembers(chanspace): + i = i.decode() + p.srem("live.who."+name+"."+i, oldnick) + p.sadd("live.who."+name+"."+i, newnick) + if main.r.exists(chanspace): + p.rename(chanspace, newchanspace) else: - main.wholist[network] = {nick: [nick, ident, host, None, None, None]} + warn("Key doesn't exist: %s" % chanspace) + p.execute() -def getWho(nick): - result = {} - for i in main.wholist.keys(): - for x in main.wholist[i].keys(): - if nick.lower() == x.lower(): - if not i in result.keys(): - result[i] = [] - result[i].append(main.wholist[i][x]) - return result +def delUserByNick(name, channel, nick): + gnamespace = "live.who.%s" % name + user = getUserByNick(name, nick) + delUser(name, channel, nick, user) + +def delUserByNetwork(name, nick, user): + gnamespace = "live.who.%s" % name + chanspace = "live.chan.%s.%s" % (name, nick) + p = main.r.pipeline() + p.srem(gnamespace, user) + for i in main.r.smembers(chanspace): + p.srem("live.who."+name+"."+i.decode(), nick) + p.delete(chanspace) + p.execute() + +def delChannel(name, channel): + gnamespace = "live.who.%s" % name + namespace = "live.who.%s.%s" % (name, channel) + p = main.r.pipeline() + for i in main.r.smembers(namespace): + user = getUserByNick(name, i.decode()) + if main.r.smembers("live.chan."+name+"."+i.decode()) == {channel.encode()}: + if user: + p.srem(gnamespace, user) + p.delete("live.chan."+name+"."+i.decode()) + else: + p.srem("live.chan."+name+"."+i.decode(), channel) + p.delete(namespace) + p.execute()