Merge branch 'master' into datarestructure

.gitignore

@@ -5,7 +5,6 @@ __pycache__/
 conf/config.json
 conf/wholist.json
 conf/counters.json
-conf/masterbuf.json
 conf/monitor.json
 conf/tokens.json
 conf/network.dat

conf/example/config.json

@@ -15,19 +15,15 @@
     "RedisSocket": "/tmp/redis.sock",
     "UsePassword": true,
     "ConnectOnCreate": false,
-    "Notifications": {
-        "Highlight": true,
-        "Connection": true,
-        "Query": true
-    },
-    "Compat": {
-        "ZNC": true
-    },
     "Dist": {
         "Enabled": true,
         "SendOutput": false,
         "File": "conf/dist.sh"
     },
+    "Toggles": {
+        "Who": false,
+        "CycleChans": true
+    },
     "Password": "s",
     "Tweaks": {
         "ZNC": {
@@ -42,6 +38,5 @@
             "Factor": 2.718281828459045,
             "Jitter": 0.11962656472
         }
-    },
+    }
-    "Master": [null, null]
 }

conf/help.json

@@ -3,8 +3,7 @@
     "logout": "logout",
     "del": "del <network> <relay>",
     "mod": "mod <network> <variable> <value>",
-    "get": "get <name>",
+    "get": "get <name> <variable>",
-    "key": "key <master|list|add|del|except|unexcept|listexcept|monitor> [<name>] [<target>] [<key...>] [<on|off>]",
     "who": "who <query>",
     "join": "join <network> <relay> <channel> [<key>]",
     "part": "part <network> <relay> <channel>",

conf/masterbuf.json

@@ -0,0 +1 @@
+[]

core/bot.py

@@ -6,6 +6,7 @@ from twisted.internet import reactor
 
 from string import digits
 from random import randint
+from copy import deepcopy
 
 from modules import userinfo
 from modules import counters as count
@@ -74,8 +75,7 @@ class IRCRelay(IRCClient):
     def signedOn(self):
         self.connected = True
         log("signed on as a relay: %s" % self.relay)
-        if main.config["Notifications"]["Connection"]:
+        #sendRelayNotification("Relay", {"type": "conn", "status": "connected"})  nobody actually cares
-            sendRelayNotification("Relay", {"type": "conn", "status": "connected"}) 
         for i in self.relayCommands.keys():
             for x in self.relayCommands[i]:
                 self.msg(main.config["Tweaks"]["ZNC"]["Prefix"]+i, x)
@@ -131,29 +131,69 @@ class IRCBot(IRCClient):
 
         return [nick, ident, host]
 
+    def event(self, **cast):
+        for i in list(cast.keys()): # Make a copy of the .keys() as Python 3 cannot handle iterating over
+            if cast[i] == "":       #  a dictionary that changes length with each iteration
+                del cast[i]
+        if "muser" in cast.keys():
+            cast["nick"], cast["ident"], cast["host"] = self.parsen(cast["muser"])
+            del cast["muser"]
+        if set(["nick", "ident", "host", "message"]).issubset(set(cast)):
+            if "message" in cast.keys():
+                if cast["ident"] == "znc" and cast["host"] == "znc.in":
+                    cast["type"] = "znc"
+                    cast["name"] = self.name
+                    del cast["nick"]
+                    del cast["ident"]
+                    del cast["host"]
+                    del cast["target"]
+        if not cast["type"] in ["query", "self", "highlight", "znc", "who"]:
+            if "target" in cast.keys():
+                if cast["target"].lower() == self.nickname.lower():
+                    #castDup = deepcopy(cast)
+                    cast["mtype"] = cast["type"]
+                    cast["type"] = "query"
+                    cast["name"] = self.name
+                    #self.event(**castDup)
+                    # Don't call self.event for this one because queries are not events on a
+                    #  channel, but we still want to see them
+            if "user" in cast.keys():
+                if cast["user"].lower() == self.nickname.lower():
+                    castDup = deepcopy(cast)
+                    castDup["mtype"] = cast["type"]
+                    castDup["type"] = "self"
+                    castDup["name"] = self.name
+                    self.event(**castDup)
+            if "nick" in cast.keys():
+                if cast["nick"].lower() == self.nickname.lower():
+                    castDup = deepcopy(cast)
+                    castDup["mtype"] = cast["type"]
+                    castDup["type"] = "self"
+                    castDup["name"] = self.name
+                    self.event(**castDup)
+            if "message" in cast.keys() and not cast["type"] == "query": # Don't highlight queries
+                if self.nickname.lower() in cast["message"].lower():
+                    castDup = deepcopy(cast)
+                    castDup["mtype"] = cast["type"]
+                    castDup["type"] = "highlight"
+                    castDup["name"] = self.name
+                    self.event(**castDup)
+
+        if "name" not in cast.keys():
+            cast["name"] = self.net
+        if cast["type"] in ["msg", "notice", "action"]:
+            userinfo.editUser(self.net, cast["nick"]+"!"+cast["ident"]+"@"+cast["host"])
+        count.event(self.net, cast["type"])
+        monitor.event(self.net, self.name, cast)
+
     def privmsg(self, user, channel, msg):
-        nick, ident, host = self.parsen(user)
+        self.event(type="msg", muser=user, target=channel, message=msg)
-        # ZNC adds messages when no users are in the channel, messing up tracking
-        #userinfo.editUser(self.net, channel, nick, user)
-        count.event(self.net, "msg")
-        #event = None
-        #if self.nickname.lower() in msg.lower():
-        #    event = "highlight"
-        monitor.event(self.net, self.name, {"type": "msg", "nick": nick, "ident": ident, "host": host, "target": channel, "message": msg})
 
     def noticed(self, user, channel, msg):
-        nick, ident, host = self.parsen(user)
+        self.event(type="notice", muser=user, target=channel, message=msg)
-        #userinfo.editUser(self.net, channel, nick, user)
-        count.event(self.net, "notice")
-
-        monitor.event(self.net, self.name, {"type": "notice", "nick": nick, "ident": ident, "host": host, "target": channel, "message": msg})
 
     def action(self, user, channel, msg):
-        nick, ident, host = self.parsen(user)
+        self.event(type="action", muser=user, target=channel, message=msg)
-        #userinfo.editUser(self.net, channel, nick, user)
-        count.event(self.net, "action")
-
-        monitor.event(self.net, self.name, {"type": "action", "nick": nick, "ident": ident, "host": host, "target": channel, "message": msg})
 
     def get(self, var):
         try:
@@ -175,7 +215,7 @@ class IRCBot(IRCClient):
         oldnick, ident, host = self.parsen(olduser)
         userinfo.renameUser(self.net, oldnick, olduser, newnick, newnick+"!"+ident+"@"+host)
         self.nickname = newnick
-        count.event(self.net, "selfnick")
+        self.event(type="self", mtype="nick", nick=oldnick, ident=ident, host=host, user=newnick)
 
     def irc_ERR_NICKNAMEINUSE(self, prefix, params):
         self._attemptedNick = self.alterCollidedNick(self._attemptedNick)
@@ -195,7 +235,7 @@ class IRCBot(IRCClient):
 
     def irc_RPL_WHOREPLY(self, prefix, params):
         channel = params[1]
-        user = params[2]
+        ident = params[2]
         host = params[3]
         server = params[4]
         nick = params[5]
@@ -204,9 +244,8 @@ class IRCBot(IRCClient):
         if channel not in self._who:
             return
         n = self._who[channel][1]
-        n.append([nick, user, host, server, status, realname])
+        n.append([nick, nick, host, server, status, realname])
-        count.event(self.net, "whoreply")
+        self.event(type="who", nick=nick, ident=ident, host=host, realname=realname, target=channel, server=server, status=status)
-        monitor.event(self.net, self.name, {"type": "who", "nick": nick, "ident": user, "host": host, "realname": realname, "target": channel, "server": server, "status": status})
 
     def irc_RPL_ENDOFWHO(self, prefix, params):
         channel = params[1]
@@ -258,7 +297,7 @@ class IRCBot(IRCClient):
         newNicklist = []
         for i in nicklist[1]:
             for x in i:
-                f = self.sanit(x)
+                f = self.sanit(x) # need to store this as well, or potentially just do not remove it...
                 if f:
                     newNicklist.append(f)
         userinfo.initialNames(self.net, nicklist[0], newNicklist)
@@ -295,10 +334,10 @@ class IRCBot(IRCClient):
         Called when a user has quit.
         """
         nick = prefix.split('!')[0]
-        if nick == self.nickname:
+        #if nick == self.nickname:
-            self.botQuit(prefix, params[0])
+        #self.botQuit(prefix, params[0])
-        else:
+        #else:
-            self.userQuit(prefix, params[0])
+        self.userQuit(prefix, params[0])
 
     def irc_NICK(self, prefix, params):
         """
@@ -318,11 +357,8 @@ class IRCBot(IRCClient):
         channel = params[0]
         kicked = params[1]
         message = params[-1]
-        if kicked.lower() == self.nickname.lower():
+        # Checks on whether it was us that was kicked are done in userKicked
-            # Yikes!
+        self.userKicked(kicked, channel, prefix, message)
-            self.kickedFrom(channel, prefix, message)
-        else:
-            self.userKicked(kicked, channel, prefix, message)
 
     def irc_TOPIC(self, prefix, params):
         """
@@ -337,26 +373,20 @@ class IRCBot(IRCClient):
     def signedOn(self):
         self.connected = True
         log("signed on: %s" % self.name)
-        if main.config["Notifications"]["Connection"]:
+        self.event(type="conn", status="connected")
-            sendRelayNotification(self.name, {"type": "conn", "status": "connected"}) 
-        count.event(self.net, "signedon")
 
     def joined(self, channel):
         if not channel in self.channels:
             self.channels.append(channel)
         self.names(channel).addCallback(self.got_names)
         self.who(channel).addCallback(self.got_who)
-        count.event(self.net, "selfjoin")
+        if main.config["Toggles"]["Who"]:
-        if self.name == main.config["Master"][0] and channel == main.config["Master"][1]:
+            lc = LoopingCall(self.who, channel)
-            for i in range(len(main.masterbuf)):
+            self._getWho[channel] = lc
-                self.msg(channel, main.masterbuf.pop(0))
+            intrange = main.config["Tweaks"]["Delays"]["WhoRange"]
-                main.saveConf("masterbuf")
+            minint = main.config["Tweaks"]["Delays"]["WhoLoop"]
-        lc = LoopingCall(self.who, channel)
+            interval = randint(minint, minint+intrange)
-        self._getWho[channel] = lc
+            lc.start(interval)
-        intrange = main.config["Tweaks"]["Delays"]["WhoRange"]
-        minint = main.config["Tweaks"]["Delays"]["WhoLoop"]
-        interval = randint(minint, minint+intrange)
-        lc.start(interval)
 
     def botLeft(self, channel):
         if channel in self.channels:
@@ -366,79 +396,62 @@ class IRCBot(IRCClient):
             lc.stop()
             del self._getWho[channel]
         userinfo.delChannel(self.net, channel)
+        log("Can no longer cover %s, removing records" % channel)
 
     def left(self, user, channel, message):
-        nick, ident, host = self.parsen(user)
+        self.event(type="part", muser=user, target=channel, message=message)
-        count.event(self.net, "selfpart")
-        monitor.event(self.net, self.name, {"type": "part", "target": channel, "message": message})
-        monitor.event(self.net, self.name, {"type": "part", "self": True, "target": channel, "message": message})
-        self.botLeft(channel)
-
-    def kickedFrom(self, channel, kicker, message):
-        nick, ident, host = self.parsen(kicker)
-        if channel in self.channels:
-            self.channels.remove(channel)
-        count.event(self.net, "selfkick")
-        monitor.event(self.net, self.name, {"type": "kick", "nick": nick, "ident": ident, "host": host, "target": channel, "message": message})
-        monitor.event(self.net, self.name, {"type": "kick", "self": True, "nick": nick, "ident": ident, "host": host, "target": channel, "message": message})
         self.botLeft(channel)
 
     def userJoined(self, user, channel):
         nick, ident, host = self.parsen(user)
         userinfo.addUser(self.net, channel, nick, user)
-        count.event(self.net, "join")
+        self.event(type="join", nick=nick, ident=ident, host=host, target=channel)
-        monitor.event(self.net, self.name, {"type": "join", "nick": nick, "ident": ident, "host": host, "target": channel})
 
     def userLeft(self, user, channel, message):
         nick, ident, host = self.parsen(user)
         userinfo.delUser(self.net, channel, nick, user)
-        count.event(self.net, "part")
+        self.event(type="part", nick=nick, ident=ident, host=host, target=channel, message=message)
-        monitor.event(self.net, self.name, {"type": "part", "nick": nick, "ident": ident, "host": host, "target": channel, "message": message})
 
     def userQuit(self, user, quitMessage):
         nick, ident, host = self.parsen(user)
+        self.chanlessEvent(type="quit", nick=nick, ident=ident, host=host, message=quitMessage)
         userinfo.delUserByNetwork(self.net, nick, user)
-        count.event(self.net, "quit")
-
-        monitor.event(self.net, self.name, {"type": "quit", "nick": nick, "ident": ident, "host": host, "message": quitMessage})
-
-    def botQuit(self, user, quitMessage):
-        nick, ident, host = self.parsen(user)
-        userinfo.delUserByNetwork(self.net, nick, user)
-        count.event(self.net, "selfquit")
-
-        monitor.event(self.net, self.name, {"type": "quit", "nick": nick, "ident": ident, "host": host, "message": quitMessage})
-        monitor.event(self.net, self.name, {"type": "quit", "self": True, "nick": nick, "ident": ident, "host": host, "message": quitMessage})
 
     def userKicked(self, kickee, channel, kicker, message):
         nick, ident, host = self.parsen(kicker)
-        userinfo.editUser(self.net, channel, nick, kicker)
+        userinfo.editUser(self.net, kicker)
         userinfo.delUserByNick(self.net, channel, kickee)
-        count.event(self.net, "kick")
+        if kickee.lower == self.nickname.lower:
-
+            self.botLeft(channel)
-        monitor.event(self.net, self.name, {"type": "kick", "nick": nick, "ident": ident, "host": host, "target": channel, "message": message, "user": kickee})
+        self.event(type="kick", nick=nick, ident=ident, host=host, target=channel, message=message, user=kickee)
+
+    def chanlessEvent(self, **cast):
+        chans = userinfo.getChansSingle(self.net, cast["nick"])
+        if chans == None:
+            self.event(**cast)
+            return
+        for i in chans:
+            cast["target"] = i
+            self.event(**cast)
 
     def userRenamed(self, oldname, newname):
         nick, ident, host = self.parsen(oldname)
+        self.chanlessEvent(type="nick", nick=nick, ident=ident, host=host, user=newname)
         userinfo.renameUser(self.net, nick, oldname, newname, newname+"!"+ident+"@"+host)
-        count.event(self.net, "nick")
-        monitor.event(self.net, self.name, {"type": "nick", "nick": nick, "ident": ident, "host": host, "user": newname})
 
     def topicUpdated(self, user, channel, newTopic):
         nick, ident, host = self.parsen(user)
-        userinfo.editUser(self.net, channel, nick, user)
+        userinfo.editUser(self.net, user)
-        count.event(self.net, "topic")
 
-        monitor.event(self.net, self.name, {"type": "topic", "nick": nick, "ident": ident, "host": host, "target": channel, "message": newTopic})
+        self.event(type="topic", nick=nick, ident=ident, host=host, target=channel, message= newTopic)
 
     def modeChanged(self, user, channel, toset, modes, args):
         nick, ident, host = self.parsen(user)
-        userinfo.editUser(self.net, channel, nick, user)
+        userinfo.editUser(self.net, user)
-        count.event(self.net, "mode")
         argList = list(args)
         modeList = [i for i in modes]
         for a, m in zip(argList, modeList):
-            monitor.event(self.net, self.name, {"type": "mode", "nick": nick, "ident": ident, "host": host, "target": channel, "modes": m, "modeargs": a})
+            self.event(type="mode", nick=nick, ident=ident, host=host, target=channel, modes=m, status=toset, modeargs=a)
 
 class IRCBotFactory(ReconnectingClientFactory):
     def __init__(self, name, relay=None, relayCommands=None, user=None, stage2=None):
@@ -475,9 +488,7 @@ class IRCBotFactory(ReconnectingClientFactory):
         log("%s: connection lost: %s" % (self.name, error))
         if not self.relay:
             sendAll("%s: connection lost: %s" % (self.name, error))
-        if main.config["Notifications"]["Connection"]:
+            sendRelayNotification(self.name, {"type": "conn", "status": "lost", "message": error})
-            sendRelayNotification(self.name, {"type": "conn", "status": "lost", "reason": error})
-        if not self.relay:
             self.retry(connector)
         #ReconnectingClientFactory.clientConnectionLost(self, connector, reason)
 
@@ -489,9 +500,7 @@ class IRCBotFactory(ReconnectingClientFactory):
         log("%s: connection failed: %s" % (self.name, error))
         if not self.relay:
             sendAll("%s: connection failed: %s" % (self.name, error))
-        if main.config["Notifications"]["Connection"]:
+            sendRelayNotification(self.name, {"type": "conn", "status": "failed", "message": error})
-            sendRelayNotification(self.name, {"type": "conn", "status": "failed", "reason": error})
-        if not self.relay:
             self.retry(connector)
         #ReconnectingClientFactory.clientConnectionFailed(self, connector, reason)
 

core/relay.py

@@ -6,9 +6,7 @@ from datetime import datetime
 import main
 from utils.logging.log import *
 
-validTypes = ["msg", "notice", "action", "who", "part", "join", "kick", "quit", "nick", "topic", "mode", "conn", "znc", "query", "self", "highlight", "monitor", "err"]
+validTypes = ["msg", "notice", "action", "who", "part", "join", "kick", "quit", "nick", "topic", "mode", "conn", "znc", "query", "self", "highlight", "monitor", "err", "query", "self", "highlight"]
-
-selfTypes = ["query", "self", "highlight"]
 
 class Relay(Protocol):
     def __init__(self, addr):
@@ -137,7 +135,7 @@ class RelayFactory(Factory):
 def sendRelayNotification(name, cast):
     for i in main.relayConnections.keys():
         if main.relayConnections[i].authed:
-            if cast["type"] in main.relayConnections[i].subscriptions or set(main.relayConnections[i].subscriptions).issubset(cast):
+            if cast["type"] in main.relayConnections[i].subscriptions:
                 newCast = deepcopy(cast)
                 newCast["name"] = name
                 newCast["time"] = str(datetime.now())

main.py

@@ -12,14 +12,13 @@ filemap = {
     "config": ["config.json", "configuration", "json"],
     "help": ["help.json", "command help", "json"],
     "counters": ["counters.json", "counters file", "json"],
-    "masterbuf": ["masterbuf.json", "master buffer", "json"],
     "monitor": ["monitor.json", "monitoring database", "json"],
     "tokens": ["tokens.json", "authentication tokens", "json"],
     "aliasdata": ["aliasdata.json", "data for alias generation", "json"],
 
     # Binary (pickle) configs
     "network": ["network.dat", "network list", "pickle"]
-    }
+}
 
 connections = {}
 relayConnections = {}

modules/monitor.py

@@ -55,11 +55,6 @@ def event(name, numberedName, cast, event=None):
         target = cast["target"]
     else:
         target = None
-    if set(["nick", "ident", "host", "message"]).issubset(set(cast)):
-        if main.config["Compat"]["ZNC"] and "message" in cast.keys():
-            if cast["nick"][0] == main.config["Tweaks"]["ZNC"]["Prefix"] and cast["ident"] == "znc" and cast["host"] == "znc.in":
-                sendRelayNotification(numberedName, {"type": "znc", "message": cast["message"]})
-                return
 
     sendRelayNotification(name, cast)
     monitorGroups = testNetTarget(name, target)

modules/provision.py

@@ -39,6 +39,9 @@ def provisionNetworkData(relay, alias, network, host, port, security, auth, pass
     if not main.config["ConnectOnCreate"]:
         stage3commands["status"] = []
         stage3commands["status"].append("Disconnect")
+    if main.config["Toggles"]["CycleChans"]:
+        stage2commands["status"] = []
+        stage2commands["status"].append("LoadMod disconkick")
     deliverRelayCommands(relay, commands,
         stage2=[[alias+"/"+network, stage2commands],
                 [alias+"/"+network, stage3commands]])

modules/userinfo.py

@@ -76,7 +76,7 @@ def initialNames(name, channel, names):
         p.sadd("live.chan."+name+"."+i, channel)
     p.execute()
 
-def editUser(name, channel, nick, user):
+def editUser(name, user):
     gnamespace = "live.who.%s" % name
     main.r.sadd(gnamespace, user)
 
@@ -151,7 +151,7 @@ def delUserByNetwork(name, nick, user):
     p.delete(chanspace)
     p.execute()
 
-def delChannel(name, channel):
+def delChannel(name, channel): # This function is extremely expensive, look to replace
     gnamespace = "live.who.%s" % name
     namespace = "live.who.%s.%s" % (name, channel)
     p = main.r.pipeline()
@@ -171,4 +171,5 @@ def delNetwork(name, channels):
     log("Purging channels for %s" % name)
     for i in channels:
         delChannel(name, i)
+    log("Finished purging channels for %s" % name)
     return