Merge branch 'master' into datarestructure

commands/all.py

@@ -0,0 +1,23 @@
+import main
+from core.bot import deliverRelayCommands
+
+class All:
+    def __init__(self, *args):
+        self.all(*args)
+
+    def all(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
+        if authed:
+            if length > 2:
+                for i in main.pool.keys():
+                    relay = main.pool[i]["relay"]
+                    network = main.pool[i]["network"]
+                    alias = main.pool[i]["alias"]
+                    commands = {spl[1]: [" ".join(spl[2:])]}
+                    success("Sending commands to relay %s as user %s" % (relay, alias+"/"+network))
+                    deliverRelayCommands(relay, commands, user=alias+"/"+network)
+                return
+            else:
+                incUsage("all")
+                return
+        else:
+            incUsage(None)

commands/allc.py

@@ -0,0 +1,46 @@
+import main
+from core.bot import deliverRelayCommands
+
+class Allc:
+    def __init__(self, *args):
+        self.allc(*args)
+
+    def allc(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
+        if authed:
+            if length > 4:
+                targets = []
+                if spl[1] == "network":
+                    if spl[2] in main.network.keys():
+                        for i in main.pool.keys():
+                            if main.pool[i]["network"] == spl[2]:
+                                targets.append(i)
+                    else:
+                        failure("No such network: %s" % spl[2])
+                        return
+                elif spl[1] == "alias":
+                    if spl[2] in main.alias.keys():
+                        for i in main.pool.keys():
+                            if main.pool[i]["alias"] == spl[2]:
+                                targets.append(i)
+                    else:
+                        failure("No such alias: %s" % spl[2])
+                        return
+                else:
+                    incUsage("allc")
+                    return
+                if len(targets) == 0:
+                    failure("No matches found: %s" % spl[2])
+                    return
+                for i in targets:
+                    relay = main.pool[i]["relay"]
+                    network = main.pool[i]["network"]
+                    alias = main.pool[i]["alias"]
+                    commands = {spl[3]: [" ".join(spl[4:])]}
+                    success("Sending commands to relay %s as user %s" % (relay, alias+"/"+network))
+                    deliverRelayCommands(relay, commands, user=alias+"/"+network)
+                return
+            else:
+                incUsage("allc")
+                return
+        else:
+            incUsage(None)

commands/provision.py

@@ -19,22 +19,22 @@ class ProvisionCommand:
                     failure("No such network: %s" % spl[3])
                     return
 
-                if "users" in main.relay[spl[1]]:
+                #if "users" in main.relay[spl[1]]:
-                    if not spl[2] in main.relay[spl[1]]["users"]:
+                #    if not spl[2] in main.relay[spl[1]]["users"]:
-                        failure("Relay %s not provisioned for alias %s" % (spl[1], spl[2]))
+                #        failure("Relay %s not provisioned for alias %s" % (spl[1], spl[2]))
-                        return
+                #        return
-                else:
+                #else:
-                        failure("Relay %s not provisioned for alias %s" % (spl[1], spl[2]))
+                #        failure("Relay %s not provisioned for alias %s" % (spl[1], spl[2]))
-                        return
+                #        return
 
                 rtrn = provision.provisionRelayForNetwork(spl[1], spl[2], spl[3])
-                if rtrn == "PROVISIONED":
+                #if rtrn == "PROVISIONED":
-                    failure("Relay %s already provisioned for alias %s on network %s" % (spl[1], spl[2], spl[3]))
+                #    failure("Relay %s already provisioned for alias %s on network %s" % (spl[1], spl[2], spl[3]))
-                    return
+                #    return
-                elif rtrn == "DUPLICATE":
+                #elif rtrn == "DUPLICATE":
-                    failure("Instance with relay %s and alias %s already exists for network %s" % (spl[1], spl[2], spl[3]))
+                #    failure("Instance with relay %s and alias %s already exists for network %s" % (spl[1], spl[2], spl[3]))
-                    return
+                #    return
-                elif rtrn:
+                if rtrn:
                     success("Started provisioning network %s on relay %s for alias %s" % (spl[3], spl[1], spl[2]))
                     info("Instance name is %s" % rtrn)
                     return
@@ -43,10 +43,10 @@ class ProvisionCommand:
                     return
             if length == 3: # provision for relay and alias only
                 rtrn = provision.provisionRelayForAlias(spl[1], spl[2])
-                if rtrn == "PROVISIONED":
+                #if rtrn == "PROVISIONED":
-                    failure("Relay %s already provisioned for alias %s" % (spl[1], spl[2]))
+                #    failure("Relay %s already provisioned for alias %s" % (spl[1], spl[2]))
-                    return
+                #    return
-                elif rtrn:
+                if rtrn:
                     success("Started provisioning relay %s for alias %s" % (spl[1], spl[2]))
                     return
                 else:

conf/example/config.json

@@ -15,6 +15,7 @@
     "RedisSocket": "/tmp/redis.sock",
     "UsePassword": true,
     "ConnectOnCreate": false,
+    "Debug": false,
     "Dist": {
         "Enabled": true,
         "SendOutput": false,
@@ -26,6 +27,8 @@
     },
     "Password": "s",
     "Tweaks": {
+        "MaxHash": 10,
+        "DedupPrecision": 9,
         "ZNC": {
             "Prefix": "*"
         },

conf/help.json

@@ -22,5 +22,7 @@
     "network": "network <add|del|list> [<name> <address> <port> <ssl|plain> <sasl|ns|none>]",
     "provision": "provision <relay> <alias> [<network>]",
     "cmd": "cmd <network> <relay> <user> <entity> <text ...>",
-    "token": "token <add|del|list> [<key>] [<relay>]"
+    "token": "token <add|del|list> [<key>] [<relay>]",
+    "all": "all <entity> <text ...>",
+    "allc": "allc <network|alias> <(network)|(alias)> <entity> <text ...>"
 }

core/bot.py

@@ -9,10 +9,11 @@ from random import randint
 from copy import deepcopy
 
 from modules import userinfo
-from modules import counters as count
+from modules import counters
 from modules import monitor
 
 from core.relay import sendRelayNotification
+from utils.dedup import dedup
 
 import main
 from utils.logging.log import *
@@ -137,7 +138,8 @@ class IRCBot(IRCClient):
                 del cast[i]
         if "muser" in cast.keys():
             cast["nick"], cast["ident"], cast["host"] = self.parsen(cast["muser"])
-            del cast["muser"]
+            #if not cast["type"] in ["nick", "kick", "quit", "part", "join"]:
+            #    del cast["muser"]
         if set(["nick", "ident", "host", "message"]).issubset(set(cast)):
             if "message" in cast.keys():
                 if cast["ident"] == "znc" and cast["host"] == "znc.in":
@@ -148,9 +150,9 @@ class IRCBot(IRCClient):
                     del cast["host"]
                     del cast["target"]
         if not cast["type"] in ["query", "self", "highlight", "znc", "who"]:
-            if "target" in cast.keys():
+            if "target" in cast.keys() and not cast["type"] == "mode": # don't handle modes here 
-                if cast["target"].lower() == self.nickname.lower():
+                if cast["target"].lower() == self.nickname.lower():    #  as they are target == nickname
-                    #castDup = deepcopy(cast)
+                    #castDup = deepcopy(cast)                          #  however modes are not queries!
                     cast["mtype"] = cast["type"]
                     cast["type"] = "query"
                     cast["name"] = self.name
@@ -170,21 +172,21 @@ class IRCBot(IRCClient):
                     castDup["mtype"] = cast["type"]
                     castDup["type"] = "self"
                     castDup["name"] = self.name
-                    self.event(**castDup)
+                    if not cast["target"].lower() == self.nickname.lower(): # modes has been set on us directly
+                        self.event(**castDup)                               #  don't tell anyone else
             if "message" in cast.keys() and not cast["type"] == "query": # Don't highlight queries
-                if self.nickname.lower() in cast["message"].lower():
+                if not cast["message"] == None:
-                    castDup = deepcopy(cast)
+                    if self.nickname.lower() in cast["message"].lower():
-                    castDup["mtype"] = cast["type"]
+                        castDup = deepcopy(cast)
-                    castDup["type"] = "highlight"
+                        castDup["mtype"] = cast["type"]
-                    castDup["name"] = self.name
+                        castDup["type"] = "highlight"
-                    self.event(**castDup)
+                        castDup["name"] = self.name
+                        self.event(**castDup)
 
-        if "name" not in cast.keys():
+        if not "name" in cast.keys():
             cast["name"] = self.net
-        if cast["type"] in ["msg", "notice", "action"]:
+        counters.event(self.net, cast["type"])
-            userinfo.editUser(self.net, cast["nick"]+"!"+cast["ident"]+"@"+cast["host"])
+        monitor.event(self.name, cast)
-        count.event(self.net, cast["type"])
-        monitor.event(self.net, self.name, cast)
 
     def privmsg(self, user, channel, msg):
         self.event(type="msg", muser=user, target=channel, message=msg)
@@ -212,10 +214,8 @@ class IRCBot(IRCClient):
         return newnick
 
     def nickChanged(self, olduser, newnick):
-        oldnick, ident, host = self.parsen(olduser)
-        userinfo.renameUser(self.net, oldnick, olduser, newnick, newnick+"!"+ident+"@"+host)
         self.nickname = newnick
-        self.event(type="self", mtype="nick", nick=oldnick, ident=ident, host=host, user=newnick)
+        self.event(type="self", mtype="nick", muser=olduser, user=newnick)
 
     def irc_ERR_NICKNAMEINUSE(self, prefix, params):
         self._attemptedNick = self.alterCollidedNick(self._attemptedNick)
@@ -379,8 +379,8 @@ class IRCBot(IRCClient):
         if not channel in self.channels:
             self.channels.append(channel)
         self.names(channel).addCallback(self.got_names)
-        self.who(channel).addCallback(self.got_who)
         if main.config["Toggles"]["Who"]:
+            self.who(channel).addCallback(self.got_who)
             lc = LoopingCall(self.who, channel)
             self._getWho[channel] = lc
             intrange = main.config["Tweaks"]["Delays"]["WhoRange"]
@@ -395,63 +395,54 @@ class IRCBot(IRCClient):
             lc = self._getWho[channel]
             lc.stop()
             del self._getWho[channel]
-        userinfo.delChannel(self.net, channel)
+        userinfo.delChannel(self.net, channel)                    # < we do not need to deduplicate this
-        log("Can no longer cover %s, removing records" % channel)
+        #log("Can no longer cover %s, removing records" % channel)#  as it will only be matched once --
-
+                                                                  #  other bots have different nicknames so
-    def left(self, user, channel, message):
+    def left(self, user, channel, message):                       #  even if they saw it, they wouldn't react
         self.event(type="part", muser=user, target=channel, message=message)
         self.botLeft(channel)
 
     def userJoined(self, user, channel):
-        nick, ident, host = self.parsen(user)
+        self.event(type="join", muser=user, target=channel)
-        userinfo.addUser(self.net, channel, nick, user)
-        self.event(type="join", nick=nick, ident=ident, host=host, target=channel)
 
     def userLeft(self, user, channel, message):
-        nick, ident, host = self.parsen(user)
+        self.event(type="part", muser=user, target=channel, message=message)
-        userinfo.delUser(self.net, channel, nick, user)
-        self.event(type="part", nick=nick, ident=ident, host=host, target=channel, message=message)
 
     def userQuit(self, user, quitMessage):
-        nick, ident, host = self.parsen(user)
+        self.chanlessEvent({"type": "quit", "muser": user, "message": quitMessage})
-        self.chanlessEvent(type="quit", nick=nick, ident=ident, host=host, message=quitMessage)
-        userinfo.delUserByNetwork(self.net, nick, user)
 
     def userKicked(self, kickee, channel, kicker, message):
-        nick, ident, host = self.parsen(kicker)
+        if kickee.lower() == self.nickname.lower():
-        userinfo.editUser(self.net, kicker)
-        userinfo.delUserByNick(self.net, channel, kickee)
-        if kickee.lower == self.nickname.lower:
             self.botLeft(channel)
-        self.event(type="kick", nick=nick, ident=ident, host=host, target=channel, message=message, user=kickee)
+        self.event(type="kick", muser=kicker, target=channel, message=message, user=kickee)
 
-    def chanlessEvent(self, **cast):
+    def chanlessEvent(self, cast):
-        chans = userinfo.getChansSingle(self.net, cast["nick"])
+        cast["nick"], cast["ident"], cast["host"] = self.parsen(cast["muser"])
+        if dedup(self.name, cast):
+            return # stop right there sir!
+        chans = userinfo.getChanList(self.net, cast["nick"])
         if chans == None:
-            self.event(**cast)
+            error("No channels returned for chanless event: %s" % cast)
+        #    self.event(**cast) -- no, should NEVER happen
             return
-        for i in chans:
+        # getChansSingle returns all channels of the user, we only want to use
+        #  ones we have common with them
+        realChans = set(chans).intersection(set(self.channels))
+        for i in realChans:
             cast["target"] = i
             self.event(**cast)
 
     def userRenamed(self, oldname, newname):
-        nick, ident, host = self.parsen(oldname)
+        self.chanlessEvent({"type": "nick", "muser": oldname, "user": newname})
-        self.chanlessEvent(type="nick", nick=nick, ident=ident, host=host, user=newname)
-        userinfo.renameUser(self.net, nick, oldname, newname, newname+"!"+ident+"@"+host)
 
     def topicUpdated(self, user, channel, newTopic):
-        nick, ident, host = self.parsen(user)
+        self.event(type="topic", muser=user, target=channel, message= newTopic)
-        userinfo.editUser(self.net, user)
-
-        self.event(type="topic", nick=nick, ident=ident, host=host, target=channel, message= newTopic)
 
     def modeChanged(self, user, channel, toset, modes, args):
-        nick, ident, host = self.parsen(user)
-        userinfo.editUser(self.net, user)
         argList = list(args)
         modeList = [i for i in modes]
         for a, m in zip(argList, modeList):
-            self.event(type="mode", nick=nick, ident=ident, host=host, target=channel, modes=m, status=toset, modeargs=a)
+            self.event(type="mode", muser=user, target=channel, modes=m, status=toset, modeargs=a)
 
 class IRCBotFactory(ReconnectingClientFactory):
     def __init__(self, name, relay=None, relayCommands=None, user=None, stage2=None):
@@ -488,7 +479,7 @@ class IRCBotFactory(ReconnectingClientFactory):
         log("%s: connection lost: %s" % (self.name, error))
         if not self.relay:
             sendAll("%s: connection lost: %s" % (self.name, error))
-            sendRelayNotification(self.name, {"type": "conn", "status": "lost", "message": error})
+            sendRelayNotification({"type": "conn", "name": self.name, "status": "lost", "message": error})
             self.retry(connector)
         #ReconnectingClientFactory.clientConnectionLost(self, connector, reason)
 
@@ -500,7 +491,7 @@ class IRCBotFactory(ReconnectingClientFactory):
         log("%s: connection failed: %s" % (self.name, error))
         if not self.relay:
             sendAll("%s: connection failed: %s" % (self.name, error))
-            sendRelayNotification(self.name, {"type": "conn", "status": "failed", "message": error})
+            sendRelayNotification({"type": "conn", "name": self.name, "status": "failed", "message": error})
             self.retry(connector)
         #ReconnectingClientFactory.clientConnectionFailed(self, connector, reason)
 

core/relay.py

@@ -132,11 +132,10 @@ class RelayFactory(Factory):
         else:
             return
 
-def sendRelayNotification(name, cast):
+def sendRelayNotification(cast):
     for i in main.relayConnections.keys():
         if main.relayConnections[i].authed:
             if cast["type"] in main.relayConnections[i].subscriptions:
                 newCast = deepcopy(cast)
-                newCast["name"] = name
+                newCast["time"] = str(datetime.now().isoformat())
-                newCast["time"] = str(datetime.now())
                 main.relayConnections[i].send(dumps(newCast))

main.py

@@ -2,6 +2,8 @@ import json
 import pickle
 from redis import StrictRedis
 from string import digits
+from os import urandom
+
 from utils.logging.log import *
 
 configPath = "conf/"
@@ -33,6 +35,10 @@ CommandMap = {}
 runningSample = 0
 lastMinuteSample = 0
 
+# Generate 16-byte hex key for message checksums
+hashKey = urandom(16)
+lastEvents = {}
+
 def nets():
     if not "pool" in globals():
         return

modules/monitor.py

@@ -1,8 +1,11 @@
 from copy import deepcopy
 from json import dumps
+from datetime import datetime
 
 import main
 from core.relay import sendRelayNotification
+from modules import userinfo
+from utils.dedup import dedup
 
 def testNetTarget(name, target):
     called = False
@@ -50,28 +53,53 @@ def magicFunction(A, B):
     else:
         return all(A[k] in B[k] for k in set(A) & set(B)) and set(B) <= set(A)
 
-def event(name, numberedName, cast, event=None):
+def event(numName, c): # yes I'm using a short variable because otherwise it goes off the screen
-    if "target" in cast.keys():
+    if not "target" in c.keys():
-        target = cast["target"]
+        c["target"] = None
-    else:
-        target = None
 
-    sendRelayNotification(name, cast)
+    if dedup(numName, c):
-    monitorGroups = testNetTarget(name, target)
+        return
+    # metadata scraping
+    # need to check if this was received from a relay
+    #  in which case, do not do this
+    if c["type"] in ["msg", "notice", "action", "topic", "mode"]:
+        userinfo.editUser(c["name"], c["muser"])
+    elif c["type"] == "nick":
+        userinfo.renameUser(c["name"], c["nick"], c["muser"], c["user"], c["user"]+"!"+c["ident"]+"@"+c["host"])
+    elif c["type"] == "kick":
+        userinfo.editUser(c["name"], c["muser"])
+        userinfo.delUserByNick(c["name"], c["target"], c["user"])
+    elif c["type"] == "quit":
+        userinfo.delUserByNetwork(c["name"], c["nick"], c["muser"])
+    elif c["type"] == "join":
+        userinfo.addUser(c["name"], c["target"], c["nick"], c["muser"])
+    elif c["type"] == "part":
+        userinfo.delUser(c["name"], c["target"], c["nick"], c["muser"])
+
+    if "mtype" in c.keys():
+        if c["mtype"] == "nick":
+            userinfo.renameUser(c["name"], c["nick"], c["muser"], c["user"], c["user"]+"!"+c["ident"]+"@"+c["host"])
+
+    if "muser" in c.keys():
+        del c["muser"]
+    sendRelayNotification(c)
+
+    # only monitors below
+    monitorGroups = testNetTarget(c["name"], c["target"])
     if monitorGroups == False:
         return
     for monitorGroup in monitorGroups:
-        matcher = magicFunction(deepcopy(cast), deepcopy(main.monitor[monitorGroup]))
+        matcher = magicFunction(deepcopy(c), deepcopy(main.monitor[monitorGroup]))
         if matcher == True:
-            cast["monitor"] = True
+            c["monitor"] = True
             if "send" in main.monitor[monitorGroup].keys():
                 for i in main.monitor[monitorGroup]["send"].keys():
                     if isinstance(main.monitor[monitorGroup]["send"][i], bool):
-                        sendRelayNotification(name, {"type": "err", "name": name, "target": target, "message": cast, "reason": "errdeliv"})
+                        sendRelayNotification({"type": "err", "name": name, "target": target, "message": c, "reason": "errdeliv"})
                         continue
                     if not i in main.pool.keys():
-                        sendRelayNotification(name, {"type": "err", "name": name, "target": target, "message": cast, "reason": "noname"})
+                        sendRelayNotification({"type": "err", "name": name, "target": target, "message": c, "reason": "noname"})
                     if not i in main.IRCPool.keys():
-                        sendRelayNotification(name, {"type": "err", "name": name, "target": target, "message": cast, "reason": "noinstance"})
+                        sendRelayNotification({"type": "err", "name": name, "target": target, "message": c, "reason": "noinstance"})
                     for x in main.monitor[monitorGroup]["send"][i]:
-                        main.IRCPool[i].msg(x, "monitor [%s] (%s) %s" % (monitorGroup, name, cast))
+                        main.IRCPool[i].msg(x, "monitor [%s] (%s) %s" % (monitorGroup, c["name"], c))

modules/provision.py

@@ -34,7 +34,7 @@ def provisionNetworkData(relay, alias, network, host, port, security, auth, pass
     elif auth == "ns":
         stage2commands["status"] = []
         stage2commands["nickserv"] = []
-        stage2commands["status"].append("LoadMod NickServ")
+        stage2commands["status"].append("LoadMod nickserv")
         stage2commands["nickserv"].append("Set %s" % password)
     if not main.config["ConnectOnCreate"]:
         stage3commands["status"] = []
@@ -42,46 +42,47 @@ def provisionNetworkData(relay, alias, network, host, port, security, auth, pass
     if main.config["Toggles"]["CycleChans"]:
         stage2commands["status"] = []
         stage2commands["status"].append("LoadMod disconkick")
+    stage2commands["status"].append("LoadMod chansaver")
     deliverRelayCommands(relay, commands,
         stage2=[[alias+"/"+network, stage2commands],
                 [alias+"/"+network, stage3commands]])
     return
 
 def provisionRelayForAlias(relay, alias):
-    if "users" in main.relay[relay].keys():
+    #if "users" in main.relay[relay].keys():
-        if alias in main.relay[relay]["users"]:
+    #    if alias in main.relay[relay]["users"]:
-            return "PROVISIONED"
+    #        return "PROVISIONED"
-    else:
+    #else:
-        main.relay[relay]["users"] = []
+    #    main.relay[relay]["users"] = []
-    main.relay[relay]["users"].append(alias)
+    #main.relay[relay]["users"].append(alias)
     provisionUserData(relay, alias, main.alias[alias]["nick"],
             main.alias[alias]["altnick"],
             main.alias[alias]["ident"],
             main.alias[alias]["realname"],
             main.relay[relay]["password"])
-    main.saveConf("relay")
+    #main.saveConf("relay")
     return True
 
 def provisionRelayForNetwork(relay, alias, network):
-    if set(["users", "networks"]).issubset(main.relay[relay].keys()):
+    #if set(["users", "networks"]).issubset(main.relay[relay].keys()):
-        if network in main.relay[relay]["networks"] and alias in main.relay[relay]["users"]:
+    #    if network in main.relay[relay]["networks"] and alias in main.relay[relay]["users"]:
-            return "PROVISIONED"
+    #        return "PROVISIONED"
-    else:
+    #else:
-        main.relay[relay]["networks"] = []
+    #    main.relay[relay]["networks"] = []
-    main.relay[relay]["networks"].append(network)
+    #main.relay[relay]["networks"].append(network)
     provisionNetworkData(relay, alias, network,
             main.network[network]["host"],
             main.network[network]["port"],
             main.network[network]["security"],
             main.network[network]["auth"],
             main.alias[alias]["password"])
-    main.saveConf("relay")
+    #main.saveConf("relay")
     storedNetwork = False
     num = 1
     while not storedNetwork:
         i = str(num)
         if num == 1000:
-            error("Too many iterations in while trying to choose name for r: %s a: %s n: %s" % (relay, alias, network))
+            error("Iteration limit exceeded while trying to choose name for r: %s a: %s n: %s" % (relay, alias, network))
             return False
 
         if network+i in main.pool.keys():

modules/userinfo.py

@@ -1,6 +1,9 @@
-import main
+from twisted.internet.threads import deferToThread
 from string import digits
+
+import main
 from utils.logging.log import *
+from utils.logging.debug import debug
 
 def getWhoSingle(name, query):
     result = main.r.sscan("live.who."+name, 0, query, count=9999999)
@@ -23,6 +26,13 @@ def getChansSingle(name, nick):
         return None
     return [i.decode() for i in result]
 
+def getChanList(name, nick):
+    chanspace =  "live.chan."+name+"."+nick
+    result = main.r.smembers(chanspace)
+    if len(result) == 0:
+        return None
+    return [i.decode() for i in result]
+
 def getChans(nick):
     result = {}
     for i in main.nets():
@@ -61,14 +71,19 @@ def getNamespace(name, channel, nick):
     chanspace = "live.chan.%s.%s" % (name, nick)
     return [gnamespace, namespace, chanspace]
 
-def initialUsers(name, channel, users):
+def _initialUsers(name, channel, users):
     gnamespace = "live.who.%s" % name
     p = main.r.pipeline()
     for i in users:
         p.sadd(gnamespace, i[0]+"!"+i[1]+"@"+i[2])
     p.execute()
 
-def initialNames(name, channel, names):
+def initialUsers(name, channel, users):
+    debug("Initialising WHO records for %s on %s" % (channel, name))
+    d = deferToThread(_initialUsers, name, channel, users)
+    #d.addCallback(testCallback)
+
+def _initialNames(name, channel, names):
     namespace = "live.who.%s.%s" % (name, channel)
     p = main.r.pipeline()
     for i in names:
@@ -76,6 +91,11 @@ def initialNames(name, channel, names):
         p.sadd("live.chan."+name+"."+i, channel)
     p.execute()
 
+def initialNames(name, channel, names):
+    debug("Initialising NAMES records for %s on %s" % (channel, name))
+    d = deferToThread(_initialNames, name, channel, names)
+    #d.addCallback(testCallback)
+
 def editUser(name, user):
     gnamespace = "live.who.%s" % name
     main.r.sadd(gnamespace, user)
@@ -151,7 +171,7 @@ def delUserByNetwork(name, nick, user):
     p.delete(chanspace)
     p.execute()
 
-def delChannel(name, channel): # This function is extremely expensive, look to replace
+def _delChannel(name, channel): # This function is extremely expensive, look to replace
     gnamespace = "live.who.%s" % name
     namespace = "live.who.%s.%s" % (name, channel)
     p = main.r.pipeline()
@@ -166,10 +186,16 @@ def delChannel(name, channel): # This function is extremely expensive, look to r
             p.srem("live.chan."+name+"."+i.decode(), channel)
     p.delete(namespace)
     p.execute()
+    return [name, channel]
+
+def delChannel(name, channel):
+    debug("Purging channel %s for %s" % (channel, name))
+    d = deferToThread(_delChannel, name, channel)
+    #d.addCallback(testCallback)
 
 def delNetwork(name, channels):
-    log("Purging channels for %s" % name)
+    debug("Purging channels for %s" % name)
     for i in channels:
         delChannel(name, i)
-    log("Finished purging channels for %s" % name)
+    #log("Finished purging channels for %s" % name)
     return

threshold

@@ -1,14 +1,19 @@
 #!/usr/bin/env python
 from twisted.internet import reactor
 from twisted.internet.ssl import DefaultOpenSSLContextFactory
+from sys import argv, stdout, stderr
 #from twisted.python import log
 #from sys import stdout
 #log.startLogging(stdout)
-
+from codecs import getwriter        # fix printing odd shit to the terminal
+stdout = getwriter("utf8")(stdout)  #  this is a generic fix but we all know
+stderr = getwriter("utf8")(stderr)  #  it's just for the retards on Rizon using
+                                    #  unicode quit messages for no reason
 import main
 
 main.initMain()
-
+if "--debug" in argv:      # yes really
+    main.config["Debug"] = True
 from utils.logging.log import *
 from utils.loaders.command_loader import loadCommands
 from core.server import Server, ServerFactory

utils/dedup.py

@@ -0,0 +1,21 @@
+from datetime import datetime
+from csiphash import siphash24
+from json import dumps
+import main
+from utils.logging.debug import debug
+
+def dedup(numName, c):
+    # deduplication
+    c["approxtime"] = str(datetime.utcnow().timestamp())[:main.config["Tweaks"]["DedupPrecision"]]
+    castHash = siphash24(main.hashKey, dumps(c, sort_keys=True).encode("utf-8"))
+    del c["approxtime"]
+    isDuplicate= any(castHash in main.lastEvents[x] for x in main.lastEvents.keys() if not x == numName)
+    if isDuplicate:
+        debug("Duplicate: %s" % (c))
+        return True
+    if numName in main.lastEvents.keys():
+        main.lastEvents[numName].insert(0, castHash)
+        main.lastEvents[numName] = main.lastEvents[numName][0:main.config["Tweaks"]["MaxHash"]]
+    else:
+        main.lastEvents[numName] = [castHash]
+    return False

utils/loaders/command_loader.py

@@ -1,6 +1,6 @@
 from os import listdir
 
-from utils.logging.log import *
+from utils.logging.debug import debug
 import commands
 
 from main import CommandMap

utils/logging/debug.py

@@ -0,0 +1,7 @@
+import main
+# we need a seperate module to log.py, as log.py is imported by main.py, and we need to access main
+#  to read the setting
+def debug(data):
+    if main.config["Debug"]:
+        print("[DEBUG]", data)
+

utils/logging/log.py

@@ -1,9 +1,6 @@
 def log(data):
     print("[LOG]", data)
 
-def debug(data):
-    print("[DEBUG]", data)
-
 def warn(data):
     print("[WARNING]", data)