Improvements to query and self event detection, implement all command and debug flags

commands/all.py

@@ -0,0 +1,23 @@
+import main
+from core.bot import deliverRelayCommands
+
+class All:
+    def __init__(self, *args):
+        self.all(*args)
+
+    def all(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
+        if authed:
+            if length > 2:
+                for i in main.pool.keys():
+                    relay = main.pool[i]["relay"]
+                    network = main.pool[i]["network"]
+                    alias = main.pool[i]["alias"]
+                    commands = {spl[1]: [" ".join(spl[2:])]}
+                    success("Sending commands to relay %s as user %s" % (relay, alias+"/"+network))
+                    deliverRelayCommands(relay, commands, user=alias+"/"+network)
+                return
+            else:
+                incUsage("all")
+                return
+        else:
+            incUsage(None)

conf/example/config.json

@@ -15,6 +15,7 @@
     "RedisSocket": "/tmp/redis.sock",
     "UsePassword": true,
     "ConnectOnCreate": false,
+    "Debug": false,
     "Dist": {
         "Enabled": true,
         "SendOutput": false,
@@ -26,6 +27,7 @@
     },
     "Password": "s",
     "Tweaks": {
+        "MaxHash": 10,
         "ZNC": {
             "Prefix": "*"
         },

conf/help.json

@@ -24,5 +24,6 @@
     "network": "network <add|del|list> [<name> <address> <port> <ssl|plain> <sasl|ns|none>]",
     "provision": "provision <relay> <alias> [<network>]",
     "cmd": "cmd <relay> <user> <entity> <text ...>",
-    "token": "token <add|del|list> [<key>] [<relay>]"
+    "token": "token <add|del|list> [<key>] [<relay>]",
+    "all": "all <entity> <text ...>"
 }

core/bot.py

@@ -9,10 +9,11 @@ from random import randint
 from copy import deepcopy
 
 from modules import userinfo
-from modules import counters as count
+from modules import counters
 from modules import monitor
 
 from core.relay import sendRelayNotification
+from utils.dedup import dedup
 
 import main
 from utils.logging.log import *
@@ -151,9 +152,9 @@ class IRCBot(IRCClient):
                     del cast["host"]
                     del cast["target"]
         if not cast["type"] in ["query", "self", "highlight", "znc", "who"]:
-            if "target" in cast.keys():
+            if "target" in cast.keys() and not cast["type"] == "mode": # don't handle modes here 
-                if cast["target"].lower() == self.nickname.lower():
+                if cast["target"].lower() == self.nickname.lower():    #  as they are target == nickname
-                    #castDup = deepcopy(cast)
+                    #castDup = deepcopy(cast)                          #  however modes are not queries!
                     cast["mtype"] = cast["type"]
                     cast["type"] = "query"
                     cast["name"] = self.name
@@ -173,7 +174,8 @@ class IRCBot(IRCClient):
                     castDup["mtype"] = cast["type"]
                     castDup["type"] = "self"
                     castDup["name"] = self.name
-                    self.event(**castDup)
+                    if not cast["target"].lower() == self.nickname.lower(): # modes has been set on us directly
+                        self.event(**castDup)                       #  don't tell anyone else
             if "message" in cast.keys() and not cast["type"] == "query": # Don't highlight queries
                 if self.nickname.lower() in cast["message"].lower():
                     castDup = deepcopy(cast)
@@ -184,8 +186,8 @@ class IRCBot(IRCClient):
 
         if not "name" in cast.keys():
             cast["name"] = self.net
-        count.event(self.net, cast["type"])
+        counters.event(self.net, cast["type"])
-        monitor.event(cast)
+        monitor.event(self.name, cast)
 
     def privmsg(self, user, channel, msg):
         self.event(type="msg", muser=user, target=channel, message=msg)
@@ -408,37 +410,40 @@ class IRCBot(IRCClient):
         self.event(type="part", muser=user, target=channel, message=message)
 
     def userQuit(self, user, quitMessage):
-        self.chanlessEvent(type="quit", muser=user, message=quitMessage)
+        self.chanlessEvent({"type": "quit", "muser": user, "message": quitMessage})
 
     def userKicked(self, kickee, channel, kicker, message):
         if kickee.lower() == self.nickname.lower():
             self.botLeft(channel)
         self.event(type="kick", muser=kicker, target=channel, message=message, user=kickee)
 
-    def chanlessEvent(self, **cast):
+    def chanlessEvent(self, cast):
-        chans = userinfo.getChansSingle(self.net, cast["nick"])
+        cast["nick"], cast["ident"], cast["host"] = self.parsen(cast["muser"])
+        if dedup(self.name, cast):
+            return # stop right there sir!
+        chans = userinfo.getChanList(self.net, cast["nick"])
         if chans == None:
-            self.event(**cast)
+            error("No channels returned for chanless event: %s" % cast)
+        #    self.event(**cast) -- no, should NEVER happen
             return
         # getChansSingle returns all channels of the user, we only want to use
         #  ones we have common with them
         realChans = set(chans).intersection(set(self.channels))
-        for i in chans:
+        for i in realChans:
             cast["target"] = i
             self.event(**cast)
 
     def userRenamed(self, oldname, newname):
-        self.chanlessEvent(type="nick", muser=oldname, user=newname)
+        self.chanlessEvent({"type": "nick", "muser": oldname, "user": newname})
 
     def topicUpdated(self, user, channel, newTopic):
         self.event(type="topic", muser=user, target=channel, message= newTopic)
 
     def modeChanged(self, user, channel, toset, modes, args):
-        nick, ident, host = self.parsen(user)
         argList = list(args)
         modeList = [i for i in modes]
         for a, m in zip(argList, modeList):
-            self.event(type="mode", nick=nick, ident=ident, host=host, target=channel, modes=m, status=toset, modeargs=a)
+            self.event(type="mode", muser=user, target=channel, modes=m, status=toset, modeargs=a)
 
 class IRCBotFactory(ReconnectingClientFactory):
     def __init__(self, name, relay=None, relayCommands=None, user=None, stage2=None):

main.py

@@ -1,6 +1,8 @@
 from json import load, dump, loads
 from redis import StrictRedis
 from string import digits
+from os import urandom
+
 from utils.logging.log import *
 
 configPath = "conf/"
@@ -31,6 +33,10 @@ CommandMap = {}
 runningSample = 0
 lastMinuteSample = 0
 
+# Generate 16-byte hex key for message checksums
+hashKey = urandom(16)
+lastEvents = {}
+
 def nets():
     if not "pool" in globals():
         return

modules/monitor.py

@@ -1,9 +1,11 @@
 from copy import deepcopy
 from json import dumps
+from datetime import datetime
 
 import main
 from core.relay import sendRelayNotification
 from modules import userinfo
+from utils.dedup import dedup
 
 def testNetTarget(name, target):
     called = False
@@ -51,10 +53,12 @@ def magicFunction(A, B):
     else:
         return all(A[k] in B[k] for k in set(A) & set(B)) and set(B) <= set(A)
 
-def event(c): # yes I'm using a short variable because otherwise it goes off the screen
+def event(numName, c): # yes I'm using a short variable because otherwise it goes off the screen
     if not "target" in c.keys():
         c["target"] = None
 
+    if dedup(numName, c):
+        return
     # metadata scraping
     # need to check if this was received from a relay
     #  in which case, do not do this
@@ -68,9 +72,9 @@ def event(c): # yes I'm using a short variable because otherwise it goes off the
     elif c["type"] == "quit":
         userinfo.delUserByNetwork(c["name"], c["nick"], c["muser"])
     elif c["type"] == "join":
-        userinfo.addUser(c["name"], c["target"], c["nick"], c["user"])
+        userinfo.addUser(c["name"], c["target"], c["nick"], c["muser"])
     elif c["type"] == "part":
-        userinfo.delUser(c["name"], c["target"], c["nick"], c["user"])
+        userinfo.delUser(c["name"], c["target"], c["nick"], c["muser"])
 
     if "mtype" in c.keys():
         if c["mtype"] == "nick":
@@ -79,6 +83,8 @@ def event(c): # yes I'm using a short variable because otherwise it goes off the
     if "muser" in c.keys():
         del c["muser"]
     sendRelayNotification(c)
+
+    # only monitors below
     monitorGroups = testNetTarget(c["name"], c["target"])
     if monitorGroups == False:
         return

modules/provision.py

@@ -34,7 +34,7 @@ def provisionNetworkData(relay, alias, network, host, port, security, auth, pass
     elif auth == "ns":
         stage2commands["status"] = []
         stage2commands["nickserv"] = []
-        stage2commands["status"].append("LoadMod NickServ")
+        stage2commands["status"].append("LoadMod nickserv")
         stage2commands["nickserv"].append("Set %s" % password)
     if not main.config["ConnectOnCreate"]:
         stage3commands["status"] = []
@@ -42,6 +42,7 @@ def provisionNetworkData(relay, alias, network, host, port, security, auth, pass
     if main.config["Toggles"]["CycleChans"]:
         stage2commands["status"] = []
         stage2commands["status"].append("LoadMod disconkick")
+    stage2commands["status"].append("LoadMod chansaver")
     deliverRelayCommands(relay, commands,
         stage2=[[alias+"/"+network, stage2commands],
                 [alias+"/"+network, stage3commands]])

modules/userinfo.py

@@ -23,6 +23,13 @@ def getChansSingle(name, nick):
         return None
     return [i.decode() for i in result]
 
+def getChanList(name, nick):
+    chanspace =  "live.chan."+name+"."+nick
+    result = main.r.smembers(chanspace)
+    if len(result) == 0:
+        return None
+    return [i.decode() for i in result]
+
 def getChans(nick):
     result = {}
     for i in main.nets():

threshold

@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 from twisted.internet import reactor
 from twisted.internet.ssl import DefaultOpenSSLContextFactory
+from sys import argv
 #from twisted.python import log
 #from sys import stdout
 #log.startLogging(stdout)
@@ -8,7 +9,8 @@ from twisted.internet.ssl import DefaultOpenSSLContextFactory
 import main
 
 main.initMain()
-
+if "--debug" in argv:      # yes really
+    main.config["Debug"] = True
 from utils.logging.log import *
 from utils.loaders.command_loader import loadCommands
 from core.helper import startBot

utils/dedup.py

@@ -0,0 +1,21 @@
+from datetime import datetime
+from csiphash import siphash24
+from json import dumps
+import main
+from utils.logging.debug import debug
+
+def dedup(numName, c):
+    # deduplication
+    c["approxtime"] = int(datetime.utcnow().timestamp())
+    castHash = siphash24(main.hashKey, dumps(c, sort_keys=True).encode("utf-8"))
+    del c["approxtime"]
+    isDuplicate= any(castHash in main.lastEvents[x] for x in main.lastEvents.keys() if not x == numName)
+    if isDuplicate:
+        debug("Duplicate: %s" % (c))
+        return True
+    if numName in main.lastEvents.keys():
+        main.lastEvents[numName].insert(0, castHash)
+        main.lastEvents[numName] = main.lastEvents[numName][0:main.config["Tweaks"]["MaxHash"]]
+    else:
+        main.lastEvents[numName] = [castHash]
+    return False

utils/loaders/command_loader.py

@@ -1,6 +1,6 @@
 from os import listdir
 
-from utils.logging.log import *
+from utils.logging.debug import debug
 import commands
 
 from main import CommandMap

utils/logging/debug.py

@@ -0,0 +1,7 @@
+import main
+# we need a seperate module to log.py, as log.py is imported by main.py, and we need to access main
+#  to read the setting
+def debug(data):
+    if main.config["Debug"]:
+        print("[DEBUG]", data)
+

utils/logging/log.py

@@ -1,9 +1,6 @@
 def log(data):
     print("[LOG]", data)
 
-def debug(data):
-    print("[DEBUG]", data)
-
 def warn(data):
     print("[WARNING]", data)