Pathogen
/
monolith
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Implement indexing into Apache Druid #1

Closed
m wants to merge 263 commits from druid into master
pull from: druid
merge into: Pathogen:master
Conversation 1 Commits 263 Files Changed 128 +1 -351
7 changed files with 1 additions and 351 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 4c08225a50 - Show all commits

1
.gitignore vendored
View File

@ -5,7 +5,6 @@ __pycache__/
conf/config.json
conf/wholist.json
conf/counters.json
conf/monitor.json
conf/tokens.json
conf/network.dat
conf/alias.json

280
commands/mon.py
View File

@ -1,280 +0,0 @@
import main
import argparse
import sys
from io import StringIO
from yaml import dump
class MonCommand:
def __init__(self, *args):
self.mon(*args)
def setup_arguments(self, ArgumentParser):
self.parser = ArgumentParser(prog="mon", description="Manage monitors. Extremely flexible. All arguments are optional.")
group1 = self.parser.add_mutually_exclusive_group(required=True)
group1.add_argument("-a", "--add", metavar="entry", dest="addEntry", help="Add an entry")
group1.add_argument("-d", "--del", metavar="entry", dest="delEntry", help="Delete an entry")
group1.add_argument("-l", "--list", action="store_true", dest="listEntry", help="List all entries")
group1.add_argument("-m", "--mod", metavar="entry", dest="modEntry", help="Modify an entry")
group2 = self.parser.add_mutually_exclusive_group()
group2.add_argument("-p", "--append", action="store_true", dest="doAppend", help="Append entries to lists instead of replacing")
group2.add_argument("-r", "--remove", action="store_true", dest="doRemove", help="Remove entries in lists instead of replacing")
self.parser.add_argument("--inside", metavar="inside", dest="inside", help="Use x in y logic for matching instead of comparing exact values")
self.parser.add_argument("--type", nargs="*", metavar="type", dest="specType", help="Specify type of spec matching. Available types: join, part, quit, msg, topic, mode, nick, kick, notice, action, who")
self.parser.add_argument("--free", nargs="*", metavar="query", dest="free", help="Use freeform matching")
self.parser.add_argument("--nick", nargs="*", metavar="nickname", dest="nick", help="Use nickname matching")
self.parser.add_argument("--ident", nargs="*", metavar="ident", dest="ident", help="Use ident matching")
self.parser.add_argument("--host", nargs="*", metavar="host", dest="host", help="Use host matching")
self.parser.add_argument("--real", nargs="*", metavar="realname", dest="real", help="Use real name (GECOS) matching. Works with types: who")
self.parser.add_argument("--source", nargs="*", action="append", metavar=("network", "channel"), dest="source", help="Target network and channel. Works with types: join, part, msg, topic, mode, kick, notice, action (can be specified multiple times)")
self.parser.add_argument("--message", nargs="*", action="append", metavar="message", dest="message", help="Message. Works with types: part, quit, msg, topic, kick, notice, action")
self.parser.add_argument("--user", nargs="*", metavar="user", dest="user", help="User (new nickname or kickee). Works with types: kick, nick")
self.parser.add_argument("--modes", nargs="*", metavar="modes", dest="modes", help="Modes. Works with types: mode")
self.parser.add_argument("--modeargs", nargs="*", metavar="modeargs", dest="modeargs", help="Mode arguments. Works with types: mode")
self.parser.add_argument("--server", nargs="*", metavar="server", dest="server", help="Server. Works with types: who")
self.parser.add_argument("--status", nargs="*", metavar="status", dest="status", help="Status. Works with types: who")
self.parser.add_argument("--send", nargs="*", action="append", metavar=("network", "target"), dest="send", help="Network and target to send notifications to (can be specified multiple times)")
def mon(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
# We need to override the ArgumentParser class because
# it's only meant for CLI applications and exits
# after running, so we override the exit function
# to do nothing. We also need to do it here in order
# to catch the error messages and show them to the user.
# It sucks. I know.
if authed:
class ArgumentParser(argparse.ArgumentParser):
def exit(self, status=0, message=None):
if message:
failure(message)
self.setup_arguments(ArgumentParser)
if length == 1:
info("Incorrect usage, use mon -h for help")
return
old_stdout = sys.stdout
old_stderr = sys.stderr
my_stdout = sys.stdout = StringIO()
my_stderr = sys.stderr = StringIO()
try:
parsed = self.parser.parse_args(spl[1:])
except:
return
sys.stdout = old_stdout
sys.stderr = old_stderr
stdout = my_stdout.getvalue()
stderr = my_stdout.getvalue()
if not stdout == "":
info(stdout)
elif not stderr == "":
failure(my_stdout.getvalue())
return
my_stdout.close()
my_stderr.close()
if parsed.addEntry:
if parsed.addEntry in main.monitor.keys():
failure("Monitor group already exists: %s, specify --mod to change" % parsed.addEntry)
return
cast = self.makeCast(parsed, failure, info)
if cast == False:
return
main.monitor[parsed.addEntry] = cast
main.saveConf("monitor")
success("Successfully created monitor group %s" % parsed.addEntry)
return
elif parsed.delEntry:
if not parsed.delEntry in main.monitor.keys():
failure("No such monitor group: %s" % parsed.delEntry)
return
del main.monitor[parsed.delEntry]
main.saveConf("monitor")
success("Successfully removed monitor group: %s" % parsed.delEntry)
return
elif parsed.modEntry:
if not parsed.doAppend and not parsed.doRemove:
failure("Specify --append or --remove with --mod")
return
if not parsed.modEntry in main.monitor.keys():
failure("No such monitor group: %s" % parsed.modEntry)
return
cast = self.makeCast(parsed, failure, info)
if cast == False:
return
if parsed.doAppend:
merged = self.addCast(main.monitor[parsed.modEntry], cast, info)
main.monitor[parsed.modEntry] = merged
elif parsed.doRemove:
merged = self.subtractCast(main.monitor[parsed.modEntry], cast, info)
if merged == {}:
del main.monitor[parsed.modEntry]
info("Group %s deleted due to having no attributes" % parsed.modEntry)
main.saveConf("monitor")
return
main.monitor[parsed.modEntry] = merged
main.saveConf("monitor")
success("Successfully updated entry %s" % parsed.modEntry)
return
elif parsed.listEntry:
info(dump(main.monitor))
return
else:
incUsage(None)
def dedup(self, data):
if not isinstance(data, bool):
return list(set(data))
return data
def parseNetworkFormat(self, lst, failure, info):
cast = {}
if lst == None:
return "nil"
if len(lst) == 0:
failure("List has no entries")
return False
elif len(lst) > 0:
if len(lst[0]) == 0:
failure("Nested list has no entries")
return False
for i in lst:
if not i[0] in main.pool.keys():
failure("Name does not exist: %s" % i[0])
return False
if i[0] in main.IRCPool.keys():
for x in i[1:]:
if not x in main.IRCPool[i[0]].channels:
info("%s: Bot not on channel: %s" % (i[0], x))
if len(i) == 1:
cast[i[0]] = True
else:
if i[0] in cast.keys():
if not cast[i[0]] == True:
for x in self.dedup(i[1:]):
cast[i[0]].append(x)
else:
cast[i[0]] = self.dedup(i[1:])
else:
cast[i[0]] = self.dedup(i[1:])
for i in cast.keys():
deduped = self.dedup(cast[i])
cast[i] = deduped
return cast
# Create or modify a monitor group magically
def makeCast(self, obj, failure, info):
validTypes = ["join", "part", "quit", "msg", "topic", "mode", "nick", "kick", "notice", "action", "who"]
cast = {}
if not obj.specType == None:
types = self.dedup(obj.specType)
for i in types:
if not i in validTypes:
failure("Invalid type: %s" % i)
info("Available types: %s" % ", ".join(validTypes))
return False
cast["type"] = types
if not obj.source == None:
sourceParse = self.parseNetworkFormat(obj.source, failure, info)
if not sourceParse:
return False
if not sourceParse == "nil":
cast["sources"] = sourceParse
if not obj.send == None:
sendParse = self.parseNetworkFormat(obj.send, failure, info)
if not sendParse:
return False
if not sendParse == "nil":
cast["send"] = sendParse
if not obj.message == None:
cast["message"] = []
for i in obj.message:
cast["message"].append(" ".join(i))
if not obj.user == None:
cast["user"] = obj.user
if not obj.modes == None:
cast["modes"] = obj.modes
if not obj.modeargs == None:
cast["modeargs"] = obj.modeargs
if not obj.server == None:
cast["server"] = obj.server
if not obj.status == None:
cast["status"] = obj.status
if not obj.free == None:
cast["free"] = obj.free
if not obj.nick == None:
cast["nick"] = obj.nick
if not obj.ident == None:
cast["ident"] = obj.ident
if not obj.host == None:
cast["host"] = obj.host
if not obj.real == None:
cast["real"] = []
for i in obj.real:
cast["real"].append(" ".join(i))
if not obj.inside == None:
if obj.inside.lower() in ["yes", "true", "on"]:
cast["inside"] = True
elif obj.inside.lower() in ["no", "false", "off"]:
cast["inside"] = False
else:
failure("inside: Unknown operand: %s" % obj.inside)
return
return cast
def subtractCast(self, source, patch, info):
for i in patch.keys():
if i in source.keys():
if isinstance(source[i], dict):
result = self.subtractCast(source[i], patch[i], info)
if result == {}:
info("Removing upper element: %s" % i)
del source[i]
continue
source[i] = result
continue
if isinstance(patch[i], bool):
del source[i]
info("Removing entire element: %s" % i)
continue
for x in patch[i]:
if isinstance(source[i], bool):
info("Attempt to remove %s from network-wide definition" % x)
continue
if x in source[i]:
source[i].remove(x)
if source[i] == []:
del source[i]
else:
info("Element %s not in source %s" % (x, i))
else:
info("Non-matched key: %s" % i)
return source
def addCast(self, source, patch, info):
for i in patch.keys():
if i in source.keys():
if isinstance(source[i], dict):
result = self.addCast(source[i], patch[i], info)
source[i] = result
continue
if isinstance(patch[i], bool):
source[i] = patch[i]
info("Overriding local element %s with network-wide definition" % i)
continue
for x in patch[i]:
if isinstance(source[i], bool):
source[i] = []
info("Overriding element %s, previously set network-wide" % i)
if x in source[i]:
info("Element %s already in source %s" % (x, i))
else:
source[i].append(x)
else:
source[i] = patch[i]
return source

1
conf/example/masterbuf.json
View File

@ -1 +0,0 @@
[]

1
conf/example/monitor.json
View File

@ -1 +0,0 @@
{}

2
core/relay.py
View File

@ -5,7 +5,7 @@ from copy import deepcopy
import main
from utils.logging.log import *
validTypes = ["msg", "notice", "action", "who", "part", "join", "kick", "quit", "nick", "topic", "mode", "conn", "znc", "query", "self", "highlight", "monitor", "err", "query", "self", "highlight"]
validTypes = ["msg", "notice", "action", "who", "part", "join", "kick", "quit", "nick", "topic", "mode", "conn", "znc", "query", "self", "highlight"]
class Relay(Protocol):
def __init__(self, addr):

1
main.py
View File

@ -20,7 +20,6 @@ filemap = {
"config": ["config.json", "configuration", "json"],
"help": ["help.json", "command help", "json"],
"counters": ["counters.json", "counters file", "json"],
"monitor": ["monitor.json", "monitoring database", "json"],
"tokens": ["tokens.json", "authentication tokens", "json"],
"aliasdata": ["aliasdata.json", "data for alias generation", "json"],
"alias": ["alias.json", "provisioned alias data", "json"],

66
modules/monitor.py
View File

@ -11,52 +11,6 @@ order = ["type", "net", "num", "channel", "msg", "nick",
"ident", "host", "mtype", "user", "mode", "modearg",
"realname", "server", "status", "time"]
def testNetTarget(name, target):
called = False
for i in main.monitor.keys():
if "sources" in main.monitor[i].keys():
if name in main.monitor[i]["sources"]:
if main.monitor[i]["sources"][name] == True:
yield i
called = True
elif target in main.monitor[i]["sources"][name]:
yield i
called = True
else:
yield i
called = True
if not called:
return False
def contained_in(x, y):
if x is None or y is None:
return False
elif x == y:
return True
else:
return y in x
def is_in(k, vs, A):
return any(contained_in(A.get(k), vp) for vp in vs)
def matches(A, B):
return all(is_in(k, vs, A) for (k, vs) in B.items())
def magicFunction(A, B):
isInside = False
if "send" in B.keys():
del B["send"]
if "sources" in B.keys():
del B["sources"]
if "inside" in B.keys():
if B["inside"] == True:
isInside = True
del B["inside"]
if isInside:
return matches(A, B)
else:
return all(A[k] in B[k] for k in set(A) & set(B)) and set(B) <= set(A)
def event(numName, c): # yes I'm using a short variable because otherwise it goes off the screen
if not "channel" in c.keys():
c["channel"] = None
@ -89,23 +43,3 @@ def event(numName, c): # yes I'm using a short variable because otherwise it goe
del c["muser"]
sendRelayNotification({k: c[k] for k in order if k in c}) # Sort dict keys according to order
# only monitors below
monitorGroups = testNetTarget(c["net"], c["channel"])
if monitorGroups == False:
return
for monitorGroup in monitorGroups:
matcher = magicFunction(deepcopy(c), deepcopy(main.monitor[monitorGroup]))
if matcher == True:
c["monitor"] = True
if "send" in main.monitor[monitorGroup].keys():
for i in main.monitor[monitorGroup]["send"].keys():
if isinstance(main.monitor[monitorGroup]["send"][i], bool):
sendRelayNotification({"type": "err", "net": net, "channel": channel, "message": c, "reason": "errdeliv"})
continue
if not i in main.pool.keys():
sendRelayNotification({"type": "err", "net": net, "channel": channel, "message": c, "reason": "noname"})
if not i in main.IRCPool.keys():
sendRelayNotification({"type": "err", "net": net, "channel": channel, "message": c, "reason": "noinstance"})
for x in main.monitor[monitorGroup]["send"][i]:
main.IRCPool[i].msg(x, "monitor [%s] (%s) %s" % (monitorGroup, c["net"], c))