Move user metadata info into redis

commands/stats.py

@@ -1,5 +1,6 @@
 import main
 import modules.counters as count
+import modules.userinfo as userinfo
 
 class Stats:
     def __init__(self, register):
@@ -13,8 +14,7 @@ class Stats:
                 numWhoEntries = 0
                 for i in main.IRCPool.keys():
                     numChannels += len(main.IRCPool[i].channels)
-                for i in main.wholist.keys():
+                numWhoEntries += userinfo.getNumTotalWhoEntries()
-                    numWhoEntries += len(main.wholist[i].keys())
                 stats.append("Servers: %s" % len(main.pool.keys()))
                 stats.append("Online servers: %s" % len(main.IRCPool.keys()))
                 stats.append("Channels: %s" % numChannels)
@@ -34,20 +34,11 @@ class Stats:
                 numChannels = 0
                 numWhoEntries = 0
 
-                failures = 0
                 if spl[1] in main.IRCPool.keys():
                     numChannels += len(main.IRCPool[spl[1]].channels)
                 else:
                     failure("Name does not exist: %s" % spl[1])
-                    failures += 1
+                numWhoEntries += userinfo.getNumWhoEntries(spl[1])
-                if spl[1] in main.wholist.keys():
-                    numWhoEntries += len(main.wholist[spl[1]].keys())
-                else:
-                    failure("Who entry does not exist: %s" % spl[1])
-                    failures += 1
-                if failures == 2:
-                    failure("No information found, aborting")
-                    return
                 stats.append("Channels: %s" % numChannels)
                 stats.append("User records: %s" % numWhoEntries)
                 counterEvents = count.getEvents(spl[1])

commands/who.py

@@ -14,9 +14,9 @@ class Who:
                     rtrn += "Matches from: %s" % i
                     rtrn += "\n"
                     for x in result[i]:
-                        x = [y for y in x if not y == None]
+                        rtrn += (x)
-                        rtrn += str((", ".join(x)))
                         rtrn += "\n"
+                    rtrn += "\n"
                 info(rtrn)
                 return
             else:

conf/help.json

@@ -14,8 +14,8 @@
     "disable": "disable <name>",
     "list": "list",
     "stats": "stats [<name>]",
-    "save": "save <config|keyconf|pool|help|wholist|counters|masterbuf|monitor|all>",
+    "save": "save <config|keyconf|pool|help|counters|masterbuf|monitor|all>",
-    "load": "load <config|keyconf|pool|help|wholist|counters|masterbuf|monitor|all>",
+    "load": "load <config|keyconf|pool|help|counters|masterbuf|monitor|all>",
     "dist": "dist",
     "loadmod": "loadmod <module>",
     "msg": "msg <name> <target> <message...>",

core/bot.py

@@ -2,6 +2,7 @@ from twisted.internet.protocol import ReconnectingClientFactory
 from twisted.words.protocols.irc import IRCClient
 from twisted.internet.defer import Deferred
 from twisted.internet.task import LoopingCall
+
 from string import digits
 from random import randint
 
@@ -19,6 +20,8 @@ class IRCBot(IRCClient):
         self.connected = False
         self.channels = []
         self.net = "".join([x for x in name if not x in digits])
+        if self.net == "":
+            error("Network with all numbers: %s" % name)
         self.buffer = ""
         self.name = name
         instance = main.pool[name]
@@ -37,6 +40,8 @@ class IRCBot(IRCClient):
         self._who = {}
         self._getWho = {}
 
+        self._names = {}
+
         self.authtype = instance["authtype"]
         if self.authtype == "ns":
             self.authpass = instance["password"]
@@ -71,7 +76,7 @@ class IRCBot(IRCClient):
 
     def privmsg(self, user, channel, msg):
         nick, ident, host = self.parsen(user)
-        userinfo.setWhoSingle(self.net, nick, ident, host)
+        userinfo.editUser(self.net, channel, nick, user)
         count.event(self.name, "privmsg")
 
         keyword.actKeyword(user, channel, msg, self.nickname, "MSG", self.name)
@@ -79,7 +84,7 @@ class IRCBot(IRCClient):
 
     def noticed(self, user, channel, msg):
         nick, ident, host = self.parsen(user)
-        userinfo.setWhoSingle(self.net, nick, ident, host)
+        userinfo.editUser(self.net, channel, nick, user)
         count.event(self.name, "notice")
 
         keyword.actKeyword(user, channel, msg, self.nickname, "NOTICE", self.name)
@@ -87,7 +92,7 @@ class IRCBot(IRCClient):
 
     def action(self, user, channel, msg):
         nick, ident, host = self.parsen(user)
-        userinfo.setWhoSingle(self.net, nick, ident, host)
+        userinfo.editUser(self.net, channel, nick, user)
         count.event(self.name, "action")
 
         keyword.actKeyword(user, channel, msg, self.nickname, "ACTION", self.name)
@@ -109,8 +114,10 @@ class IRCBot(IRCClient):
         newnick = nickname + "_"
         return newnick
 
-    def nickChanged(self, nick):
+    def nickChanged(self, olduser, newnick):
-        self.nickname = nick
+        oldnick, ident, host = self.parsen(olduser)
+        userinfo.renameUser(self.net, oldnick, olduser, newnick, newnick+"!"+ident+"@"+host)
+        self.nickname = newnick
         count.event(self.name, "selfnick")
 
     def irc_ERR_NICKNAMEINUSE(self, prefix, params):
@@ -122,10 +129,9 @@ class IRCBot(IRCClient):
         sendAll("%s: password mismatch" % self.name)
 
     def who(self, channel):
-        channel = channel
         d = Deferred()
         if channel not in self._who:
-            self._who[channel] = ([], {})
+            self._who[channel] = ([], [])
         self._who[channel][0].append(d)
         self.sendLine("WHO %s" % channel)
         return d
@@ -141,7 +147,7 @@ class IRCBot(IRCClient):
         if channel not in self._who:
             return
         n = self._who[channel][1]
-        n[nick] = [nick, user, host, server, status, realname]
+        n.append([nick, user, host, server, status, realname])
         count.event(self.name, "whoreply")
         monitor.event(self.net, channel, {"type": "who", "exact": nick+"!"+user+"@"+host, "nick": nick, "ident": user, "host": host, "realname": realname, "server": server, "status": status})
 
@@ -155,7 +161,50 @@ class IRCBot(IRCClient):
         del self._who[channel]
 
     def got_who(self, whoinfo):
-        userinfo.setWho(self.name, whoinfo[1])
+        userinfo.initialUsers(self.net, whoinfo[0], whoinfo[1])
+
+    def sanit(self, data):
+        if len(data) >= 1:
+            if data[0] in ["!", "~", "&", "@", "%", "+"]:
+                data = data[1:]
+                return data
+            return data
+        else:
+            return False
+
+    def names(self, channel):
+        d = Deferred()
+        if channel not in self._names:
+            self._names[channel] = ([], [])
+        self._names[channel][0].append(d)
+        self.sendLine("NAMES %s" % channel)
+        return d
+
+    def irc_RPL_NAMREPLY(self, prefix, params):
+        channel = params[2]
+        nicklist = params[3].split(' ')
+        if channel not in self._names:
+            return
+        n = self._names[channel][1]
+        n.append(nicklist)
+
+    def irc_RPL_ENDOFNAMES(self, prefix, params):
+        channel = params[1]
+        if channel not in self._names:
+            return
+        callbacks, namelist = self._names[channel]
+        for cb in callbacks:
+            cb.callback((channel, namelist))
+        del self._names[channel]
+
+    def got_names(self, nicklist):
+        newNicklist = []
+        for i in nicklist[1]:
+            for x in i:
+                f = self.sanit(x)
+                if f:
+                    newNicklist.append(f)
+        userinfo.initialNames(self.net, nicklist[0], newNicklist)
 
     #twisted sucks so i have to do this to actually get the user info
     def irc_JOIN(self, prefix, params):
@@ -197,7 +246,7 @@ class IRCBot(IRCClient):
         """
         nick = prefix.split('!', 1)[0]
         if nick == self.nickname:
-            self.nickChanged(params[0])
+            self.nickChanged(prefix, params[0])
         else:
             self.userRenamed(prefix, params[0])
 
@@ -236,61 +285,63 @@ class IRCBot(IRCClient):
             self.join(i)
         count.event(self.name, "signedon")
 
-    def getWho(self, channel):
-        self.who(channel).addCallback(self.got_who)
-
     def joined(self, channel):
         if not channel in self.channels:
             self.channels.append(channel)
+        self.names(channel).addCallback(self.got_names)
         self.who(channel).addCallback(self.got_who)
         count.event(self.name, "selfjoin")
         if self.name == main.config["Master"][0] and channel == main.config["Master"][1]:
             for i in range(len(main.masterbuf)):
                 self.msg(channel, main.masterbuf.pop(0))
                 main.saveConf("masterbuf")
-        lc = LoopingCall(self.getWho, channel)
+        lc = LoopingCall(self.who, channel)
         self._getWho[channel] = lc
         intrange = main.config["Tweaks"]["Delays"]["WhoRange"]
         minint = main.config["Tweaks"]["Delays"]["WhoLoop"]
         interval = randint(minint, minint+intrange)
         lc.start(interval)
 
-    def left(self, channel, message):
+    def botLeft(self, channel):
         if channel in self.channels:
             self.channels.remove(channel)
+        if channel in self._getWho.keys():
+            lc = self._getWho[channel]
+            lc.stop()
+            del self._getWho[channel]
+        userinfo.delChannel(self.net, channel)
+
+    def left(self, channel, message):
         keyword.actKeyword(self.nickname, channel, message, self.nickname, "SELFPART", self.name)
         count.event(self.name, "selfpart")
         monitor.event(self.net, channel, {"type": "part", "message": message})
-        lc = self._getWho[channel]
+        self.botLeft(channel)
-        lc.stop()
 
     def kickedFrom(self, channel, kicker, message):
         nick, ident, host = self.parsen(kicker)
-        userinfo.setWhoSingle(self.net, nick, ident, host)
         if channel in self.channels:
             self.channels.remove(channel)
         keyword.sendMaster("KICK %s: (%s/%s) %s" % (self.name, kicker, channel, message))
         count.event(self.name, "selfkick")
         monitor.event(self.net, channel, {"type": "kick", "exact": kicker, "nick": nick, "ident": ident, "host": host, "message": message})
-        lc = self._getWho[channel]
+        self.botLeft(channel)
-        lc.stop()
 
     def userJoined(self, user, channel):
         nick, ident, host = self.parsen(user)
-        userinfo.setWhoSingle(self.net, nick, ident, host)
+        userinfo.addUser(self.net, channel, nick, user)
         count.event(self.name, "join")
         monitor.event(self.net, channel, {"type": "join", "exact": user, "nick": nick, "ident": ident, "host": host})
 
     def userLeft(self, user, channel, message):
         nick, ident, host = self.parsen(user)
-        userinfo.setWhoSingle(self.net, nick, ident, host)
+        userinfo.delUser(self.net, channel, nick, user)
         keyword.actKeyword(user, channel, message, self.nickname, "PART", self.name)
         count.event(self.name, "part")
         monitor.event(self.net, channel, {"type": "part", "exact": user, "nick": nick, "ident": ident, "host": host, "message": message})
 
     def userQuit(self, user, quitMessage):
         nick, ident, host = self.parsen(user)
-        userinfo.setWhoSingle(self.net, nick, ident, host)
+        userinfo.delUserByNetwork(self.net, nick, user)
         count.event(self.name, "quit")
 
         keyword.actKeyword(user, None, quitMessage, self.nickname, "QUIT", self.name)
@@ -298,7 +349,8 @@ class IRCBot(IRCClient):
 
     def userKicked(self, kickee, channel, kicker, message):
         nick, ident, host = self.parsen(kicker)
-        userinfo.setWhoSingle(self.net, nick, ident, host)
+        userinfo.editUser(self.net, channel, nick, kicker)
+        userinfo.delUserByNick(self.net, channel, kickee)
         count.event(self.name, "kick")
 
         keyword.actKeyword(kicker, channel, message, self.nickname, "KICK", self.name)
@@ -306,14 +358,13 @@ class IRCBot(IRCClient):
 
     def userRenamed(self, oldname, newname):
         nick, ident, host = self.parsen(oldname)
-        userinfo.setWhoSingle(self.net, nick, ident, host)
+        userinfo.renameUser(self.net, nick, oldname, newname, newname+"!"+ident+"@"+host)
-        userinfo.setWhoSingle(self.net, newname, ident, host)
         count.event(self.name, "nick")
         monitor.event(self.net, None, {"type": "nick", "exact": oldname, "nick": nick, "ident": ident, "host": host, "user": newname})
 
     def topicUpdated(self, user, channel, newTopic):
         nick, ident, host = self.parsen(user)
-        userinfo.setWhoSingle(self.net, nick, ident, host)
+        userinfo.editUser(self.net, channel, nick, user)
         count.event(self.name, "topic")
 
         keyword.actKeyword(user, channel, newTopic, self.nickname, "TOPIC", self.name)
@@ -321,7 +372,7 @@ class IRCBot(IRCClient):
 
     def modeChanged(self, user, channel, toset, modes, args):
         nick, ident, host = self.parsen(user)
-        userinfo.setWhoSingle(self.net, nick, ident, host)
+        userinfo.editUser(self.net, channel, nick, user)
         count.event(self.name, "mode")
         argList = list(args)
         modeList = [i for i in modes]

main.py

@@ -1,8 +1,10 @@
 from json import load, dump, loads
-
+import redis
 from utils.loaders.command_loader import loadCommands
 from utils.logging.log import *
 
+r = redis.StrictRedis(unix_socket_path='/tmp/redis.sock', db=0)
+
 configPath = "conf/"
 certPath = "cert/"
 
@@ -11,7 +13,7 @@ filemap = {
     "keyconf": ["keyword.json", "keyword lists"],
     "pool": ["pool.json", "pool"],
     "help": ["help.json", "command help"],
-    "wholist": ["wholist.json", "WHO lists"],
+#    "wholist": ["wholist.json", "WHO lists"],
     "counters": ["counters.json", "counters file"],
     "masterbuf": ["masterbuf.json", "master buffer"],
     "monitor": ["monitor.json", "monitoring database"],

modules/keyword.py

@@ -29,7 +29,6 @@ def sendMaster(data):
         if not hasMonitors:
             main.masterbuf.append(data)
             main.saveConf("masterbuf")
-        warn("Master with no IRC instance defined")
 
     for i in main.MonitorPool:
         main.connections[i].send(data)

modules/userinfo.py

@@ -1,31 +1,131 @@
 import main
 from string import digits
-#from utils.logging.log import *
+from utils.logging.log import *
 
-def setWho(network, newObjects):
+def getWhoSingle(name, query):
-    if not network in main.wholist.keys():
+    result = main.r.sscan("live.who."+name, 0, query)
-        main.wholist[network] = {}
+    if result[1] == []:
-    for i in newObjects.keys():
+        return None
-        main.wholist[network][i] = newObjects[i]
+    return [i.decode() for i in result[1]]
 
-    return
+def getWho(query):
-
-def setWhoSingle(network, nick, ident, host):
-    if network in main.wholist.keys():
-        if nick in main.wholist[network].keys():
-            main.wholist[network][nick][1] = ident
-            main.wholist[network][nick][2] = host
-        else:
-            main.wholist[network][nick] = [nick, ident, host, None, None, None]
-    else:
-        main.wholist[network] = {nick: [nick, ident, host, None, None, None]}
-
-def getWho(nick):
     result = {}
-    for i in main.wholist.keys():
+    for i in main.pool.keys():
-        for x in main.wholist[i].keys():
+        f = getWhoSingle(i, query)
-            if nick.lower() == x.lower():
+        if f:
-                if not i in result.keys():
+            result[i] = f
-                    result[i] = []
+    print(result)
-                result[i].append(main.wholist[i][x])
     return result
+
+def getNumWhoEntries(name):
+    return main.r.scard("live.who."+name)
+
+def getNumTotalWhoEntries():
+    total = 0
+    for i in main.pool.keys():
+        total += getNumWhoEntries(i)
+    return total
+
+def getNamespace(name, channel, nick):
+    gnamespace = "live.who.%s" % name
+    namespace = "live.who.%s.%s" % (name, channel)
+    chanspace = "live.chan.%s.%s" % (name, nick)
+    return [gnamespace, namespace, chanspace]
+
+def initialUsers(name, channel, users):
+    gnamespace = "live.who.%s" % name
+    p = main.r.pipeline()
+    for i in users:
+        p.sadd(gnamespace, i[0]+"!"+i[1]+"@"+i[2])
+    p.execute()
+
+def initialNames(name, channel, names):
+    namespace = "live.who.%s.%s" % (name, channel)
+    p = main.r.pipeline()
+    for i in names:
+        p.sadd(namespace, i)
+        p.sadd("live.chan."+name+"."+i, channel)
+    p.execute()
+
+def editUser(name, channel, nick, user):
+    gnamespace = "live.who.%s" % name
+    main.r.sadd(gnamespace, user)
+
+def addUser(name, channel, nick, user):
+    gnamespace, namespace, chanspace = getNamespace(name, channel, nick)
+    p = main.r.pipeline()
+    p.sadd(gnamespace, user)
+    p.sadd(namespace, nick)
+    p.sadd(chanspace, channel)
+    p.execute()
+
+def delUser(name, channel,  nick, user):
+    gnamespace, namespace, chanspace = getNamespace(name, channel, nick)
+    p = main.r.pipeline()
+    channels = main.r.smembers(chanspace)
+    p.srem(namespace, nick)
+    if channels == {channel.encode()}:
+        p.delete(chanspace)
+        p.srem(gnamespace, user)
+    else:
+        p.srem(chanspace, channel)
+    p.execute()
+
+def getUserByNick(name, nick):
+    gnamespace = "live.who.%s" % name
+    usermatch = main.r.sscan(gnamespace, match=nick+"!*")
+    if usermatch[1] == []:
+        return False
+    else:
+        if len(usermatch[1]) == 1:
+            user = usermatch[1][0]
+            return user
+        else:
+            return False
+
+def renameUser(name, oldnick, olduser, newnick, newuser):
+    gnamespace = "live.who.%s" % name
+    chanspace = "live.chan.%s.%s" % (name, oldnick)
+    newchanspace = "live.chan.%s.%s" % (name, newnick)
+    p = main.r.pipeline()
+    p.srem(gnamespace, olduser)
+    p.sadd(gnamespace, newuser)
+    for i in main.r.smembers(chanspace):
+        i = i.decode()
+        p.srem("live.who."+name+"."+i, oldnick)
+        p.sadd("live.who."+name+"."+i, newnick)
+    if main.r.exists(chanspace):
+        p.rename(chanspace, newchanspace)
+    else:
+        warn("Key doesn't exist: %s" % chanspace)
+    p.execute()
+
+def delUserByNick(name, channel, nick):
+    gnamespace = "live.who.%s" % name
+    user = getUserByNick(name, nick)
+    delUser(name, channel, nick, user)
+
+def delUserByNetwork(name, nick, user):
+    gnamespace = "live.who.%s" % name
+    chanspace = "live.chan.%s.%s" % (name, nick)
+    p = main.r.pipeline()
+    p.srem(gnamespace, user)
+    for i in main.r.smembers(chanspace):
+        p.srem("live.who."+name+"."+i.decode(), nick)
+    p.delete(chanspace)
+    p.execute()
+
+def delChannel(name, channel):
+    gnamespace = "live.who.%s" % name
+    namespace = "live.who.%s.%s" % (name, channel)
+    p = main.r.pipeline()
+    for i in main.r.smembers(namespace):
+        user = getUserByNick(name, i.decode())
+        if main.r.smembers("live.chan."+name+"."+i.decode()) == {channel.encode()}:
+            if user:
+                p.srem(gnamespace, user)
+            p.delete("live.chan."+name+"."+i.decode())
+        else:
+            p.srem("live.chan."+name+"."+i.decode(), channel)
+    p.delete(namespace)
+    p.execute()