Implement indexing into Apache Druid #1

Closed
m wants to merge 263 commits from druid into master
4 changed files with 30 additions and 43 deletions
Showing only changes of commit 778690ae3a - Show all commits

View File

@ -7,8 +7,6 @@ class LogoutCommand:
def logout(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
if authed:
obj.authed = False
if obj.addr in main.MonitorPool:
main.MonitorPool.remove(obj.addr)
success("Logged out")
return
else:

View File

@ -38,6 +38,7 @@ def deliverRelayCommands(num, relayCommands, user=None, stage2=None):
port,
bot, contextFactory)
# Copied from the Twisted source so we can fix a bug
def parsemsg(s):
"""
Breaks a message from an IRC server into its prefix, command, and
@ -55,10 +56,10 @@ def parsemsg(s):
prefix, s = s[1:].split(' ', 1)
if s.find(' :') != -1:
s, trailing = s.split(' :', 1)
args = s.split(' ')
args = s.split(' ') # Twisted bug fixed by adding an argument to split()
args.append(trailing)
else:
args = s.split(' ')
args = s.split(' ') # And again
command = args.pop(0)
return prefix, command, args
@ -107,7 +108,6 @@ class IRCRelay(IRCClient):
if not self.isconnected:
self.isconnected = True
log("signed on as a relay: %s" % self.num)
#sendRelayNotification("Relay", {"type": "conn", "status": "connected"}) nobody actually cares
sleeptime = 0
increment = 0.8
for i in self.relayCommands.keys():
@ -135,7 +135,7 @@ class IRCBot(IRCClient):
self.password = main.config["Relay"]["Password"]
self.userinfo = None #
self.fingerReply = None #
self.versionName = None # don't tell anyone information about us
self.versionName = None # Don't give out information
self.versionNum = None #
self.versionEnv = None #
self.sourceURL = None #
@ -369,7 +369,7 @@ class IRCBot(IRCClient):
newNicklist = []
for i in nicklist[1]:
for x in i:
f = self.sanit(x) # need to store this as well, or potentially just do not remove it...
f = self.sanit(x)
if f:
newNicklist.append(f)
userinfo.initialNames(self.net, nicklist[0], newNicklist)
@ -477,15 +477,13 @@ class IRCBot(IRCClient):
if len(split) == 2:
chanlimit = split[1]
break
print("chanlimit", chanlimit)
try:
self.chanlimit = int(chanlimit)
except TypeError:
warn("Invalid chanlimit: %s" % i)
if self.chanlimit == 0:
self.chanlimit = 200 # don't take the piss if it's unlimited
self.chanlimit = 200 # don't take the piss if it's not limited
allRelays = chankeep.allRelaysActive(self.net)
print(self.net, self.num, allRelays)
if allRelays:
for i in main.network.keys():
for x in main.network[i].relays.keys():
@ -503,11 +501,10 @@ class IRCBot(IRCClient):
debug("Aborting LIST due to bad chanlimit")
self.checkChannels()
#twisted sucks so i have to do this to actually get the user info
# We need to override these functions as Twisted discards
# the hostname and other useful information in the functions
# that these call by default
def irc_JOIN(self, prefix, params):
"""
Called when a user joins a channel.
"""
nick = prefix.split('!')[0]
channel = params[-1]
if nick == self.nickname:
@ -516,9 +513,6 @@ class IRCBot(IRCClient):
self.userJoined(prefix, channel)
def irc_PART(self, prefix, params):
"""
Called when a user leaves a channel.
"""
nick = prefix.split('!')[0]
channel = params[0]
if len(params) >= 2:
@ -531,19 +525,10 @@ class IRCBot(IRCClient):
self.userLeft(prefix, channel, message)
def irc_QUIT(self, prefix, params):
"""
Called when a user has quit.
"""
nick = prefix.split('!')[0]
#if nick == self.nickname:
#self.botQuit(prefix, params[0])
#else:
self.userQuit(prefix, params[0])
def irc_NICK(self, prefix, params):
"""
Called when a user changes their nickname.
"""
nick = prefix.split('!', 1)[0]
if nick == self.nickname:
self.nickChanged(prefix, params[0])
@ -551,10 +536,6 @@ class IRCBot(IRCClient):
self.userRenamed(prefix, params[0])
def irc_KICK(self, prefix, params):
"""
Called when a user is kicked from a channel.
"""
#kicker = prefix.split('!')[0]
channel = params[0]
kicked = params[1]
message = params[-1]
@ -562,18 +543,13 @@ class IRCBot(IRCClient):
self.userKicked(kicked, channel, prefix, message)
def irc_TOPIC(self, prefix, params):
"""
Someone in the channel set the topic.
"""
#user = prefix.split('!')[0]
channel = params[0]
newtopic = params[1]
self.topicUpdated(prefix, channel, newtopic)
#END hacks
# End of Twisted hackery
def signedOn(self):
log("signed on: %s - %i" % (self.net, self.num))
#self.event(type="conn", status="connected")
sendRelayNotification({"type": "conn", "net": self.net, "num": self.num, "status": "signedon"})
def joined(self, channel):
@ -581,8 +557,6 @@ class IRCBot(IRCClient):
self.channels.append(channel)
self.names(channel).addCallback(self.got_names)
if main.config["Toggles"]["Who"]:
#self.who(channel).addCallback(self.got_who)
#self.who(channel)
lc = LoopingCall(self.who, channel)
self._getWho[channel] = lc
intrange = main.config["Tweaks"]["Delays"]["WhoRange"]

View File

@ -38,8 +38,6 @@ class Server(Protocol):
del main.connections[self.addr]
else:
warn("Tried to remove a non-existant connection.")
if self.addr in main.MonitorPool:
main.MonitorPool.remove(self.addr)
class ServerFactory(Factory):
def buildProtocol(self, addr):

23
main.py
View File

@ -6,6 +6,7 @@ from os import urandom
from utils.logging.log import *
# List of errors ZNC can give us
ZNCErrors = ["Error:",
"Unable to load",
"does not exist",
@ -28,17 +29,32 @@ filemap = {
"network": ["network.dat", "network list", "pickle"]
}
# Connections to the plain-text interface
connections = {}
# Connections to the JSON interface
relayConnections = {}
# Mapping of network names to Protocol (IRCClient) instances
IRCPool = {}
# Mapping of network names to Reactor instances
# Needed for calling .disconnect()
ReactorPool = {}
# Mapping of network names to Factory instances
# Needed for calling .stopTrying()
FactoryPool = {}
# Temporary store for channels allocated after a LIST
# Will get purged as the instances fire up and pop() from
# their respective keys in here
TempChan = {}
MonitorPool = []
# Mapping of command names to their functions
CommandMap = {}
# Incremented by 1 for each event reaching modules.counters.event()
# and cloned into lastMinuteSample every minute
runningSample = 0
lastMinuteSample = 0
@ -46,6 +62,7 @@ lastMinuteSample = 0
hashKey = urandom(16)
lastEvents = {}
# Get networks that are currently online and dedupliate
def liveNets():
networks = set()
for i in IRCPool.keys():
@ -67,7 +84,7 @@ def loadConf(var):
with open(configPath+filemap[var][0], "r") as f:
globals()[var] = json.load(f)
if var == "alias":
# This is a hack to convert all the keys into integers since JSON
# This is a workaround to convert all the keys into integers since JSON
# turns them into strings...
# Dammit Jason!
global alias