Pathogen
/
monolith
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Implement indexing into Apache Druid #1

Closed
m wants to merge 263 commits from druid into master
pull from: druid
merge into: Pathogen:master
Conversation 1 Commits 263 Files Changed 128 +116 -85
5 changed files with 116 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit f50a40d207 - Show all commits

155
core/bot.py
View File

@ -125,9 +125,9 @@ class IRCRelay(IRCClient):
class IRCBot(IRCClient):
def __init__(self, net, num):
self.isconnected = False
self.authenticated = False
self.channels = []
self.net = net
self.authenticated = not regproc.needToRegister(self.net)
self.num = num
self.buffer = ""
self.name = net + str(num)
@ -230,6 +230,7 @@ class IRCBot(IRCClient):
del cast["ident"]
del cast["host"]
del cast["channel"]
del cast["muser"]
if "Disconnected from IRC" in cast["msg"]:
log("ZNC disconnected on %s - %i" % (self.net, self.num))
self.isconnected = False
@ -244,16 +245,16 @@ class IRCBot(IRCClient):
cast["num"] = self.num
if "channel" in cast.keys():
if cast["type"] == "mode":
if self.nickname.lower() == cast["channel"].lower():
castDup = deepcopy(cast)
castDup["mtype"] = cast["type"]
castDup["type"] = "self"
self.event(**castDup)
if cast["modearg"]:
if cast["channel"].lower() == self.nickname.lower():
#castDup = deepcopy(cast)
cast["mtype"] = cast["type"]
cast["type"] = "self"
#self.event(**castDup)
if cast["modearg"]: # check if modearg is non-NoneType
if self.nickname.lower() == cast["modearg"].lower():
castDup = deepcopy(cast)
castDup["mtype"] = cast["type"]
castDup["type"] = "self"
castDup["type"] = "highlight"
self.event(**castDup)
else:
if cast["channel"].lower() == self.nickname.lower():
@ -263,34 +264,39 @@ class IRCBot(IRCClient):
# Don't call self.event for this one because queries are not events on a
# channel, but we still want to see them
# we have been kicked
if "user" in cast.keys():
if cast["user"].lower() == self.nickname.lower():
castDup = deepcopy(cast)
castDup["mtype"] = cast["type"]
castDup["type"] = "self"
self.event(**castDup)
# we sent a message/left/joined/kick someone/quit
if "nick" in cast.keys():
if cast["nick"].lower() == self.nickname.lower():
castDup = deepcopy(cast)
castDup["mtype"] = cast["type"]
castDup["type"] = "self"
# we have been mentioned in a msg/notice/action/part/quit/topic message
if "msg" in cast.keys(): # Don't highlight queries
if not cast["msg"] == None:
if self.nickname.lower() in cast["msg"].lower():
# TODO: better way to do this
# as we changed the types above, check again
if not cast["type"] in {"query", "self", "highlight", "znc", "who"}:
# we have been kicked
if "user" in cast.keys():
if cast["user"].lower() == self.nickname.lower():
castDup = deepcopy(cast)
castDup["mtype"] = cast["type"]
castDup["type"] = "highlight"
castDup["type"] = "self"
self.event(**castDup)
# we sent a message/left/joined/kick someone/quit
if "nick" in cast.keys():
if cast["nick"].lower() == self.nickname.lower():
castDup = deepcopy(cast)
castDup["mtype"] = cast["type"]
castDup["type"] = "self"
# we have been mentioned in a msg/notice/action/part/quit/topic message
if "msg" in cast.keys(): # Don't highlight queries
if not cast["msg"] == None:
if self.nickname.lower() in cast["msg"].lower():
castDup = deepcopy(cast)
castDup["mtype"] = cast["type"]
castDup["type"] = "highlight"
self.event(**castDup)
if not "net" in cast.keys():
cast["net"] = self.net
if not "num" in cast.keys():
cast["num"] = self.num
if not self.authenticated:
regproc.registerTest(cast)
counters.event(self.net, cast["type"])
monitor.event(self.net, cast)
@ -429,8 +435,8 @@ class IRCBot(IRCClient):
return d
def list(self, noargs=False, nocheck=False):
if not main.network[self.net].relays[self.num]["registered"]:
debug("Will not send LIST, unregistered: %s - %i" % (self.net, self.num))
if not self.authenticated:
debug("Will not send LIST, unauthenticated: %s - %i" % (self.net, self.num))
return
if self.listAttempted:
debug("List request dropped, already asked for LIST - %s - %i" % (self.net, self.num))
@ -496,45 +502,15 @@ class IRCBot(IRCClient):
return
chankeep.initialList(self.net, self.num, listinfo, self.chanlimit)
def isupport(self, options):
interested = ("CHANLIMIT", "MAXCHANNELS")
if not any((x for x in options if any(y in x for y in interested))):
return # check if any of interested is in any of options, some networks
chanlimit = None # call isupport() more than once, so discard swiftly anything
if not self.isconnected: # we don't care about
log("endpoint connected: %s - %i" % (self.net, self.num))
self.isconnected = True
if not main.network[self.net].relays[self.num]["registered"]:
if main.config["AutoReg"]:
self._regAttempt = reactor.callLater(5, regproc.registerAccount, self.net, self.num)
#regproc.registerAccount(self.net, self.num)
for i in options:
if i.startswith("CHANLIMIT"):
if ":" in i:
split = i.split(":")
if len(split) >= 2:
chanlimit = split[1]
break
elif i.startswith("MAXCHANNELS"):
if "=" in i:
split = i.split("=")
if len(split) == 2:
chanlimit = split[1]
break
try:
self.chanlimit = int(chanlimit)
except TypeError:
warn("Invalid chanlimit: %s" % i)
if self.chanlimit == 0:
self.chanlimit = 200 # don't take the piss if it's not limited
def recheckList(self):
print("list being rechecked")
allRelays = chankeep.allRelaysActive(self.net)
if allRelays:
for i in main.network.keys():
for x in main.network[i].relays.keys():
name = i+str(x)
if main.IRCPool[name].wantList == True:
main.IRCPool[name].list(nocheck=True)
debug("Asking for a list for %s after final relay %i connected" % (self.net, self.num))
print("allrelays now passed")
name = self.net+"1"
if main.IRCPool[name].wantList == True:
main.IRCPool[name].list(nocheck=True)
debug("Asking for a list for %s after final relay %i connected" % (self.net, self.num))
if self.num == 1: # Only one instance should do a list
if self.chanlimit:
if allRelays:
@ -545,6 +521,51 @@ class IRCBot(IRCClient):
debug("Aborting LIST due to bad chanlimit")
self.checkChannels()
def seed_chanlimit(self, chanlimit):
if not main.network[self.net].relays[self.num]["registered"]: #TODO: add check for register request sent, only send it once
if main.config["AutoReg"]:
self._regAttempt = reactor.callLater(5, regproc.registerAccount, self.net, self.num)
#regproc.registerAccount(self.net, self.num)
try:
self.chanlimit = int(chanlimit)
except TypeError:
warn("Invalid chanlimit: %s" % i)
if self.chanlimit == 0:
self.chanlimit = 200 # don't take the piss if it's not limited
if not regproc.needToRegister(self.net): # if we need to register, only recheck on auth confirmation
self.recheckList()
def seed_prefix(self, prefix):
print("PREFIX", prefix)
def isupport(self, options):
interested = {"CHANLIMIT", "MAXCHANNELS", "PREFIX"}
newOptions = {x for x in options if any(y in x for y in interested)}
if len(newOptions) == 0:
return
if not self.isconnected:
log("endpoint connected: %s - %i" % (self.net, self.num))
self.isconnected = True
for i in newOptions:
if i.startswith("PREFIX"):
if "=" in i:
split = i.split("=")
if len(split) == 2:
prefix = split[1]
self.seed_prefix(prefix)
elif i.startswith("CHANLIMIT"):
if ":" in i:
split = i.split(":")
if len(split) >= 2:
chanlimit = split[1]
self.seed_chanlimit(chanlimit)
elif i.startswith("MAXCHANNELS"):
if "=" in i:
split = i.split("=")
if len(split) == 2:
chanlimit = split[1]
self.seed_chanlimit(chanlimit)
# We need to override these functions as Twisted discards
# the hostname and other useful information in the functions
# that these call by default

2
modules/chankeep.py
View File

@ -12,7 +12,7 @@ def allRelaysActive(net):
for i in main.network[net].relays.keys():
name = net+str(i)
if name in main.IRCPool.keys():
if main.IRCPool[name].authenticated and main.network[net].relays[i]["registered"]:
if main.IRCPool[name].authenticated:
existNum += 1
if existNum == relayNum:
return True

14
modules/monitor.py
View File

@ -11,13 +11,9 @@ order = ["type", "net", "num", "channel", "msg", "nick",
"ident", "host", "mtype", "user", "mode", "modearg",
"realname", "server", "status", "time"]
def event(numName, c): # yes I'm using a short variable because otherwise it goes off the screen
def parsemeta(numName, c):
if not "channel" in c.keys():
c["channel"] = None
if dedup(numName, c):
return
regproc.registerTest(c)
# metadata scraping
# need to check if this was received from a relay
# in which case, do not do this
@ -39,7 +35,13 @@ def event(numName, c): # yes I'm using a short variable because otherwise it goe
if c["mtype"] == "nick":
userinfo.renameUser(c["net"], c["nick"], c["muser"], c["user"], c["user"]+"!"+c["ident"]+"@"+c["host"])
def event(numName, c): # yes I'm using a short variable because otherwise it goes off the screen
if dedup(numName, c):
return
# make a copy of the object with dict() to prevent sending notifications with channel of None
parsemeta(numName, dict(c))
if "muser" in c.keys():
del c["muser"]
sendRelayNotification({k: c[k] for k in order if k in c}) # Sort dict keys according to order

12
modules/network.py
View File

@ -3,6 +3,7 @@ import json
from modules import alias
from modules.chankeep import nukeNetwork
from modules.regproc import needToRegister
from twisted.internet import reactor
from core.bot import IRCBot, IRCBotFactory
import main
@ -28,13 +29,10 @@ class Network:
elif num == self.last:
self.last += 1
registered = False
if self.net in main.irc.keys():
if "register" in main.irc[self.net].keys():
if not main.irc[self.net]["register"]:
registered = True
# Don't need to register if it's been disabled in definitions,
# so we'll pretend we already did
if not needToRegister(self.net):
registered = True
# Don't need to register if it's been disabled in definitions,
# so we'll pretend we already did
self.relays[num] = {
"enabled": main.config["ConnectOnCreate"],
"net": self.net,

18
modules/regproc.py
View File

@ -4,6 +4,14 @@ from utils.logging.log import *
from utils.logging.debug import *
from copy import deepcopy
def needToRegister(net):
inst = selectInst(net)
if "register" in inst.keys():
if inst["register"]:
return True
else:
return False
def selectInst(net):
if net in main.irc.keys():
inst = deepcopy(main.irc[net])
@ -48,6 +56,7 @@ def confirmRegistration(net, num):
if name in main.IRCPool.keys():
debug("Relay authenticated: %s - %i" %(net, num))
main.IRCPool[name].authenticated = True
main.IRCPool[name].recheckList()
if obj.relays[num]["registered"]:
return
if name in main.IRCPool.keys():
@ -78,7 +87,8 @@ def registerTest(c):
confirmRegistration(c["net"], c["num"])
return
elif inst["checktype"] == "mode":
if c["type"] == "mode":
if inst["checkmode"] in c["modes"] and c["status"] == True:
confirmRegistration(c["net"], c["num"])
return
if c["type"] == "self":
if c["mtype"] == "mode":
if inst["checkmode"] in c["mode"] and c["status"] == True:
confirmRegistration(c["net"], c["num"])
return