Implement monitoring system for flexible metadata matching

commands/delete.py

@@ -19,7 +19,7 @@ class Delete:
                         del main.IRCPool[spl[1]]
                     del main.ReactorPool[spl[1]]
                     del main.FactoryPool[spl[1]]
-                success("Successfully removed bot")
+                success("Successfully removed bot: %s" % spl[1])
                 main.saveConf("pool")
                 return
             else:

commands/key.py

@@ -123,11 +123,11 @@ class Key:
                     incUsage("key")
                     return
             elif length == 2:
-                if spl[1] == "show":
+                if spl[1] == "list":
                     info(",".join(main.keyconf["Keywords"]))
                     return
 
-                elif spl[1] == "showexcept":
+                elif spl[1] == "listexcept":
                     exceptMap = []
                     for i in main.keyconf["KeywordsExcept"].keys():
                         exceptMap.append("Key: %s" % i)

commands/mon.py

@@ -0,0 +1,236 @@
+import main
+import argparse
+import sys
+from io import StringIO
+from pprint import pformat
+
+class Mon:
+    def __init__(self, register):
+        register("mon", self.mon)
+
+    def setup_arguments(self, ArgumentParser):
+        self.parser = ArgumentParser(prog="mon", description="Manage monitors. Extremely flexible. All arguments are optional.")
+        group1 = self.parser.add_mutually_exclusive_group(required=True)
+        group1.add_argument("--add", metavar="entry", dest="addEntry",  help="Add an entry")
+        group1.add_argument("--del", metavar="entry", dest="delEntry", help="Delete an entry")
+        group1.add_argument("--list", action="store_true", dest="listEntry", help="List all entries")
+        group1.add_argument("--mod", metavar="entry", dest="modEntry", help="Modify an entry")
+
+        group2 = self.parser.add_mutually_exclusive_group()
+        group2.add_argument("--append", action="store_true", dest="doAppend", help="Append entries to lists instead of replacing")
+        group2.add_argument("--remove", action="store_true", dest="doRemove", help="Remove entries in lists instead of replacing")
+
+        self.parser.add_argument("--type", nargs="*", metavar="type", dest="specType", help="Specify type of spec matching. Available types: join, part, quit, msg, topic, mode, nick, kick, notice, action, any")
+        self.parser.add_argument("--free", nargs="*", metavar="query", dest="free", help="Use freeform matching")
+        self.parser.add_argument("--exact", nargs="*", metavar="query", dest="exact", help="Use exact matching")
+        self.parser.add_argument("--nick", nargs="*", metavar="nickname", dest="nick", help="Use nickname matching")
+        self.parser.add_argument("--ident", nargs="*", metavar="ident", dest="ident", help="Use ident matching")
+        self.parser.add_argument("--host", nargs="*", metavar="host", dest="host", help="Use host matching")
+        self.parser.add_argument("--real", nargs="*", metavar="realname", dest="real", help="Use real name (GECOS) matching (only works on WHO)")
+
+        self.parser.add_argument("--source", nargs="*", action="append", metavar=("network", "channel"), dest="source", help="Target network and channel. Works with types: join, part, msg, topic, mode, kick, notice, action (can be specified multiple times)")
+        self.parser.add_argument("--message", nargs="*", action="append", metavar="message", dest="message", help="Message. Works with types: part, quit, msg, topic, kick, notice, action")
+        self.parser.add_argument("--user", nargs="*", metavar="user", dest="user", help="User (new nickname or kickee). Works with types: kick, nick")
+        self.parser.add_argument("--modes", nargs="*", metavar="modes", dest="modes", help="Modes. Works with types: mode")
+        self.parser.add_argument("--send", nargs="*", action="append", metavar=("network", "target"), dest="send", help="Network and target to send notifications to (can be specified multiple times)")
+
+    def mon(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
+        # We need to override the ArgumentParser class because
+            # it's only meant for CLI applications and exits
+            # after running, so we override the exit function
+            # to do nothing. We also need to do it here in order
+            # to catch the error messages and show them to the user.
+            # It sucks. I know.
+        if authed:
+            class ArgumentParser(argparse.ArgumentParser):
+                def exit(self, status=0, message=None):
+                    if message:
+                        failure(message)
+            self.setup_arguments(ArgumentParser)
+            if length == 1:
+                info("Incorrect usage, use mon -h for help")
+                return
+            old_stdout = sys.stdout
+            old_stderr = sys.stderr
+            my_stdout = sys.stdout = StringIO()
+            my_stderr = sys.stderr = StringIO()
+            try:
+                parsed = self.parser.parse_args(spl[1:])
+            except:
+                return
+            sys.stdout = old_stdout
+            sys.stderr = old_stderr
+            stdout = my_stdout.getvalue()
+            stderr = my_stdout.getvalue()
+            if not stdout == "":
+                info(stdout)
+            elif not stderr == "":
+                failure(my_stdout.getvalue())
+                return
+            my_stdout.close()
+            my_stderr.close()
+
+            if parsed.addEntry:
+                if parsed.addEntry in main.monitor.keys():
+                    failure("Monitor group already exists: %s, specify --mod to change" % parsed.addEntry)
+                    return
+                cast = self.makeCast(parsed, failure, info)
+                if cast == False:
+                    return
+                main.monitor[parsed.addEntry] = cast
+                main.saveConf("monitor")
+                success("Successfully created monitor group %s" % parsed.addEntry)
+                return
+            elif parsed.delEntry:
+                if not parsed.delEntry in main.monitor.keys():
+                    failure("No such monitor group: %s" % parsed.delEntry)
+                    return
+                del main.monitor[parsed.delEntry]
+                main.saveConf("monitor")
+                success("Successfully removed monitor group: %s" % parsed.delEntry)
+                return
+            elif parsed.modEntry:
+                if not parsed.modEntry in main.monitor.keys():
+                    failure("No such monitor group: %s" % parsed.modEntry)
+                    return
+                cast = self.makeCast(parsed, failure, info)
+                if cast == False:
+                    return
+                if parsed.doAppend:
+                    merged = self.addCast(main.monitor[parsed.modEntry], cast, info)
+                    main.monitor[parsed.modEntry] = merged
+                elif parsed.doRemove:
+                    merged = self.subtractCast(main.monitor[parsed.modEntry], cast, info)
+                    main.monitor[parsed.modEntry] = merged
+                else:
+                    failure("Specify --append or --remove with --mod")
+                    return
+                main.saveConf("monitor")
+                success("Successfully updated entry %s" % parsed.modEntry)
+                return
+            elif parsed.listEntry:
+                info(pformat(main.monitor))
+                return
+
+        else:
+            incUsage(None)
+
+    def parseNetworkFormat(self, lst, failure, info):
+        dedup = lambda x: list(set(x))
+        cast = {}
+        if lst == None:
+            return "nil"
+        if len(lst) == 0:
+            failure("List has no entries")
+            return False
+        elif len(lst) > 0:
+            if len(lst[0]) == 0:
+                failure("Nested list has no entries")
+                return False
+        for i in lst:
+            if not i[0] in main.pool.keys():
+                failure("Name does not exist: %s" % i[0])
+                return False
+            if i[0] in main.IRCPool.keys():
+                for x in i[1:]:
+                    if not x in main.IRCPool[i[0]].channels:
+                        info("%s: Bot not on channel: %s" % (i[0], x))
+            if len(i) == 1:
+                cast[i[0]] = True
+            else:
+                if i[0] in cast.keys():
+                    if not cast[i[0]] == True:
+                        for x in dedup(i[1:]):
+                            cast[i[0]].append(x)
+                    else:
+                        cast[i[0]] = dedup(i[1:])
+                else:
+                    cast[i[0]] = dedup(i[1:])
+        for i in cast.keys():
+            deduped = dedup(cast[i])
+            cast[i] = deduped
+        return cast
+
+    # Create or modify a monitor group magically
+    def makeCast(self, obj, failure, info):
+        dedup = lambda x: list(set(x))
+        validTypes = ["join", "part", "quit", "msg", "topic", "mode", "nick", "kick", "notice", "action", "any"]
+        cast = {}
+        if not obj.specType == None:
+            types = dedup(obj.specType)
+            for i in types:
+                if not i in validTypes:
+                    failure("Invalid type: %s" % i)
+                    info("Available types: %s" % ", ".join(validTypes))
+                    return False
+            cast["type"] = types
+        if not obj.source == None:
+            sourceParse = self.parseNetworkFormat(obj.source, failure, info)
+            if not sourceParse:
+                return False
+            if not sourceParse == "nil":
+                cast["sources"] = sourceParse
+
+        if not obj.send == None:
+            sendParse = self.parseNetworkFormat(obj.send, failure, info)
+            if not sendParse:
+                return False
+            if not sendParse == "nil":
+                cast["send"] = sendParse
+
+        if not obj.message == None:
+            cast["message"] = []
+            for i in obj.message:
+                cast["message"].append(" ".join(i))
+
+        if not obj.user == None:
+            cast["user"] = obj.user
+        if not obj.modes == None:
+            cast["modes"] = obj.modes
+        if not obj.free == None:
+            cast["free"] = obj.free
+        if not obj.exact == None:
+            cast["exact"] = obj.exact
+        if not obj.nick == None:
+            cast["nick"] = obj.nick
+        if not obj.ident == None:
+            cast["ident"] = obj.ident
+        if not obj.host == None:
+            cast["host"] = obj.host
+        if not obj.real == None:
+            cast["real"] = []
+            for i in obj.real:
+                cast["real"].append(" ".join(i))
+        return cast
+
+    def subtractCast(self, source, patch, info):
+        for i in patch.keys():
+            if i in source.keys():
+                if isinstance(source[i], dict):
+                    result = self.subtractCast(source[i], patch[i], info)
+                    source[i] = result
+                    continue
+                for x in patch[i]:
+                    if x in source[i]:
+                        source[i].remove(x)
+                    else:
+                        info("Element %s not in source %s" % (x, i))
+            else:
+                info("Non-matched key: %s" % i)
+        return source
+
+    def addCast(self, source, patch, info):
+        for i in patch.keys():
+            if i in source.keys():
+                if isinstance(source[i], dict):
+                    result = self.addCast(source[i], patch[i], info)
+                    source[i] = result
+                    continue
+                for x in patch[i]:
+                    if x in source[i]:
+                        info("Element %s already in source %s" % (x, i))
+                    else:
+                        source[i].append(x)
+            else:
+                source[i] = patch[i]
+        return source

conf/example/mon.json

@@ -0,0 +1 @@
+{}

conf/example/monitor.json

@@ -0,0 +1 @@
+{}

conf/help.json

@@ -6,16 +6,18 @@
     "mod": "mod <name> [<key>] [<value>]",
     "default": "default [<key>] [<value>]",
     "get": "get <name> <variable>",
-    "key": "key <master|show|add|del|except|unexcept|showexcept|monitor> [<name>] [<target>] [<key...>] [<on|off>]",
+    "key": "key <master|list|add|del|except|unexcept|listexcept|monitor> [<name>] [<target>] [<key...>] [<on|off>]",
     "who": "who <nick>",
     "join": "join <name> <channel> [<key>]",
+    "part": "part <name> <channel>",
     "enable": "enable <name>",
     "disable": "disable <name>",
     "list": "list",
     "stats": "stats [<name>]",
-    "save": "save <config|keyconf|pool|help|wholist|counters|masterbuf|all>",
+    "save": "save <config|keyconf|pool|help|wholist|counters|masterbuf|monitor|all>",
-    "load": "load <config|keyconf|pool|help|wholist|counters|masterbuf|all>",
+    "load": "load <config|keyconf|pool|help|wholist|counters|masterbuf|monitor|all>",
     "dist": "dist",
     "loadmod": "loadmod <module>",
-    "msg": "msg <name> <target> <message...>"
+    "msg": "msg <name> <target> <message...>",
+    "mon": "mon -h"
 }

core/bot.py

@@ -5,6 +5,7 @@ from twisted.internet.defer import Deferred
 import modules.keyword as keyword
 import modules.userinfo as userinfo
 import modules.counters as count
+import modules.monitor as monitor
 
 import main
 from utils.logging.log import *
@@ -70,6 +71,7 @@ class IRCBot(IRCClient):
         count.event(self.name, "privmsg")
 
         keyword.actKeyword(user, channel, msg, self.nickname, "MSG", self.name)
+        monitor.event(self.name, channel, {"type": "msg", "exact": user, "nick": nick, "ident": ident, "host": host, "message": msg})
 
     def noticed(self, user, channel, msg):
         nick, ident, host = self.parsen(user)
@@ -77,6 +79,7 @@ class IRCBot(IRCClient):
         count.event(self.name, "notice")
 
         keyword.actKeyword(user, channel, msg, self.nickname, "NOTICE", self.name)
+        monitor.event(self.name, channel, {"type": "notice", "exact": user, "nick": nick, "ident": ident, "host": host, "message": msg})
 
     def action(self, user, channel, msg):
         nick, ident, host = self.parsen(user)
@@ -84,6 +87,7 @@ class IRCBot(IRCClient):
         count.event(self.name, "action")
 
         keyword.actKeyword(user, channel, msg, self.nickname, "ACTION", self.name)
+        monitor.event(self.name, channel, {"type": "action", "exact": user, "nick": nick, "ident": ident, "host": host, "message": msg})
 
     def get(self, var):
         try:
@@ -241,25 +245,32 @@ class IRCBot(IRCClient):
     def left(self, channel, message):
         if channel in self.channels:
             self.channels.remove(channel)
-        keyword.actKeyword(user, channel, message, self.nickname, "SELFPART", self.name)
+        keyword.actKeyword(self.nickname, channel, message, self.nickname, "SELFPART", self.name)
         count.event(self.name, "selfpart")
+        monitor.event(self.name, channel, {"type": "part", "message": message})
 
     def kickedFrom(self, channel, kicker, message):
+        nick, ident, host = self.parsen(kicker)
+        print(kicker)
+        userinfo.setWhoSingle(self.name, nick, ident, host)
         if channel in self.channels:
             self.channels.remove(channel)
         keyword.sendMaster("KICK %s: (%s/%s) %s" % (self.name, kicker, channel, message))
         count.event(self.name, "selfkick")
+        monitor.event(self.name, channel, {"type": "kick", "exact": kicker, "nick": nick, "ident": ident, "host": host, "message": message})
 
     def userJoined(self, user, channel):
         nick, ident, host = self.parsen(user)
         userinfo.setWhoSingle(self.name, nick, ident, host)
         count.event(self.name, "join")
+        monitor.event(self.name, channel, {"type": "join", "exact": user, "nick": nick, "ident": ident, "host": host})
 
     def userLeft(self, user, channel, message):
         nick, ident, host = self.parsen(user)
         userinfo.setWhoSingle(self.name, nick, ident, host)
         keyword.actKeyword(user, channel, message, self.nickname, "PART", self.name)
         count.event(self.name, "part")
+        monitor.event(self.name, channel, {"type": "part", "exact": user, "nick": nick, "ident": ident, "host": host, "message": message})
 
     def userQuit(self, user, quitMessage):
         nick, ident, host = self.parsen(user)
@@ -267,6 +278,7 @@ class IRCBot(IRCClient):
         count.event(self.name, "quit")
 
         keyword.actKeyword(user, None, quitMessage, self.nickname, "QUIT", self.name)
+        monitor.event(self.name, None, {"type": "quit", "exact": user, "nick": nick, "ident": ident, "host": host, "message": quitMessage})
 
     def userKicked(self, kickee, channel, kicker, message):
         nick, ident, host = self.parsen(kicker)
@@ -274,12 +286,14 @@ class IRCBot(IRCClient):
         count.event(self.name, "kick")
 
         keyword.actKeyword(kicker, channel, message, self.nickname, "KICK", self.name)
+        monitor.event(self.name, channel, {"type": "kick", "exact": kicker, "nick": nick, "ident": ident, "host": host, "message": message, "user": kickee})
 
     def userRenamed(self, oldname, newname):
         nick, ident, host = self.parsen(oldname)
         userinfo.setWhoSingle(self.name, nick, ident, host)
         userinfo.setWhoSingle(self.name, newname, ident, host)
         count.event(self.name, "nick")
+        monitor.event(self.name, None, {"type": "nick", "exact": oldname, "nick": nick, "ident": ident, "host": host, "user": newname})
 
     def topicUpdated(self, user, channel, newTopic):
         nick, ident, host = self.parsen(user)
@@ -287,11 +301,14 @@ class IRCBot(IRCClient):
         count.event(self.name, "topic")
 
         keyword.actKeyword(user, channel, newTopic, self.nickname, "TOPIC", self.name)
+        monitor.event(self.name, channel, {"type": "topic", "exact": user, "nick": nick, "ident": ident, "host": host, "message": newTopic})
 
     def modeChanged(self, user, channel, toset, modes, args):
         nick, ident, host = self.parsen(user)
         userinfo.setWhoSingle(self.name, nick, ident, host)
         count.event(self.name, "mode")
+        print("%s // %s // %s" % (toset, modes, args))
+        #monitor.event(self.name, channel, {"type": "topic", "exact": user, "nick": nick, "ident": ident, "host": host, "message": newTopic})
 
 class IRCBotFactory(ReconnectingClientFactory):
     def __init__(self, name):

main.py

@@ -14,6 +14,7 @@ filemap = {
     "wholist": ["wholist.json", "WHO lists"],
     "counters": ["counters.json", "counters file"],
     "masterbuf": ["masterbuf.json", "master buffer"],
+    "monitor": ["monitor.json", "monitoring database"],
     }
 
 connections = {}

modules/keyword.py

@@ -89,7 +89,7 @@ def actKeyword(user, channel, message, nickname, actType, name):
                 if main.config["Notifications"]["Query"]:
                     if user == main.config["Tweaks"]["ZNC"]["Prefix"] + "status!znc@znc.in":
                         if main.config["Compat"]["ZNC"]:
-                            sendMaster("ZNC %s %s: (%s/%s) %s" % (actType, name, user, channel, msgLower))
+                            sendMaster("ZNC %s %s: %s" % (actType, name, msgLower))
                             ZNCAlreadySent = True
                         else:
                             sendMaster("QUERY %s %s: (%s) %s" % (actType, name, user, msgLower))
@@ -98,7 +98,7 @@ def actKeyword(user, channel, message, nickname, actType, name):
                 if not ZNCAlreadySent == True:
                     if main.config["Compat"]["ZNC"]:
                         if user == main.config["Tweaks"]["ZNC"]["Prefix"] + "status!znc@znc.in":
-                            sendMaster("ZNC %s %s: (%s/%s) %s" % (actType, name, user, channel, msgLower))
+                            sendMaster("ZNC %s %s: %s" % (actType, name, msgLower))
         if toSend:
             sendMaster("MATCH %s %s (U:%s T:%s): (%s/%s) %s" % (actType, name, toSend[1], toSend[2], user, channel, toSend[0]))
             count.event(name, "keymatch")

modules/monitor.py

@@ -0,0 +1,36 @@
+import main
+import modules.keyword as keyword
+from pprint import pformat
+
+def testNetTarget(name, target):
+    for i in main.monitor.keys():
+        if "sources" in main.monitor[i].keys():
+            if name in main.monitor[i]["sources"]:
+                if main.monitor[i]["sources"][name] == True:
+                    return i
+                elif target in main.monitor[i]["sources"][name]:
+                    return i
+        else:
+            return i
+    return False
+
+def magicFunction(A, B):
+    return all(A[k] in B[k] for k in set(A) & set(B)) and set(B) <= set(A)
+
+def event(name, target, cast):
+    monitorGroup = testNetTarget(name, target)
+    if monitorGroup == False:
+        return
+    matcher = magicFunction(cast, main.monitor[monitorGroup])
+
+    if matcher == True:
+        if "send" in main.monitor[monitorGroup]:
+            for i in main.monitor[monitorGroup]["send"].keys():
+                if not i in main.pool.keys():
+                    keyword.sendMaster("ERROR on monitor %s: No such name: %s" % (monitorGroup, i))
+                if not i in main.IRCPool.keys():
+                    keyword.sendMaster("ERROR on monitor %s: No such instance: %s" % (monitorGroup, i))
+                for x in main.monitor[monitorGroup]["send"][i]:
+                    main.IRCPool[i].msg(x, "MONITOR [%s] %s" % (monitorGroup, pformat(cast)))
+        else:
+            keyword.sendMaster("MONITOR [%s] %s " % (monitorGroup, pformat(cast)))