Implement indexing into Apache Druid #1

Closed
m wants to merge 263 commits from druid into master
4 changed files with 30 additions and 43 deletions
Showing only changes of commit 778690ae3a - Show all commits

View File

@ -7,8 +7,6 @@ class LogoutCommand:
def logout(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length): def logout(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
if authed: if authed:
obj.authed = False obj.authed = False
if obj.addr in main.MonitorPool:
main.MonitorPool.remove(obj.addr)
success("Logged out") success("Logged out")
return return
else: else:

View File

@ -38,6 +38,7 @@ def deliverRelayCommands(num, relayCommands, user=None, stage2=None):
port, port,
bot, contextFactory) bot, contextFactory)
# Copied from the Twisted source so we can fix a bug
def parsemsg(s): def parsemsg(s):
""" """
Breaks a message from an IRC server into its prefix, command, and Breaks a message from an IRC server into its prefix, command, and
@ -55,10 +56,10 @@ def parsemsg(s):
prefix, s = s[1:].split(' ', 1) prefix, s = s[1:].split(' ', 1)
if s.find(' :') != -1: if s.find(' :') != -1:
s, trailing = s.split(' :', 1) s, trailing = s.split(' :', 1)
args = s.split(' ') args = s.split(' ') # Twisted bug fixed by adding an argument to split()
args.append(trailing) args.append(trailing)
else: else:
args = s.split(' ') args = s.split(' ') # And again
command = args.pop(0) command = args.pop(0)
return prefix, command, args return prefix, command, args
@ -107,7 +108,6 @@ class IRCRelay(IRCClient):
if not self.isconnected: if not self.isconnected:
self.isconnected = True self.isconnected = True
log("signed on as a relay: %s" % self.num) log("signed on as a relay: %s" % self.num)
#sendRelayNotification("Relay", {"type": "conn", "status": "connected"}) nobody actually cares
sleeptime = 0 sleeptime = 0
increment = 0.8 increment = 0.8
for i in self.relayCommands.keys(): for i in self.relayCommands.keys():
@ -135,7 +135,7 @@ class IRCBot(IRCClient):
self.password = main.config["Relay"]["Password"] self.password = main.config["Relay"]["Password"]
self.userinfo = None # self.userinfo = None #
self.fingerReply = None # self.fingerReply = None #
self.versionName = None # don't tell anyone information about us self.versionName = None # Don't give out information
self.versionNum = None # self.versionNum = None #
self.versionEnv = None # self.versionEnv = None #
self.sourceURL = None # self.sourceURL = None #
@ -369,7 +369,7 @@ class IRCBot(IRCClient):
newNicklist = [] newNicklist = []
for i in nicklist[1]: for i in nicklist[1]:
for x in i: for x in i:
f = self.sanit(x) # need to store this as well, or potentially just do not remove it... f = self.sanit(x)
if f: if f:
newNicklist.append(f) newNicklist.append(f)
userinfo.initialNames(self.net, nicklist[0], newNicklist) userinfo.initialNames(self.net, nicklist[0], newNicklist)
@ -477,15 +477,13 @@ class IRCBot(IRCClient):
if len(split) == 2: if len(split) == 2:
chanlimit = split[1] chanlimit = split[1]
break break
print("chanlimit", chanlimit)
try: try:
self.chanlimit = int(chanlimit) self.chanlimit = int(chanlimit)
except TypeError: except TypeError:
warn("Invalid chanlimit: %s" % i) warn("Invalid chanlimit: %s" % i)
if self.chanlimit == 0: if self.chanlimit == 0:
self.chanlimit = 200 # don't take the piss if it's unlimited self.chanlimit = 200 # don't take the piss if it's not limited
allRelays = chankeep.allRelaysActive(self.net) allRelays = chankeep.allRelaysActive(self.net)
print(self.net, self.num, allRelays)
if allRelays: if allRelays:
for i in main.network.keys(): for i in main.network.keys():
for x in main.network[i].relays.keys(): for x in main.network[i].relays.keys():
@ -503,11 +501,10 @@ class IRCBot(IRCClient):
debug("Aborting LIST due to bad chanlimit") debug("Aborting LIST due to bad chanlimit")
self.checkChannels() self.checkChannels()
#twisted sucks so i have to do this to actually get the user info # We need to override these functions as Twisted discards
# the hostname and other useful information in the functions
# that these call by default
def irc_JOIN(self, prefix, params): def irc_JOIN(self, prefix, params):
"""
Called when a user joins a channel.
"""
nick = prefix.split('!')[0] nick = prefix.split('!')[0]
channel = params[-1] channel = params[-1]
if nick == self.nickname: if nick == self.nickname:
@ -516,9 +513,6 @@ class IRCBot(IRCClient):
self.userJoined(prefix, channel) self.userJoined(prefix, channel)
def irc_PART(self, prefix, params): def irc_PART(self, prefix, params):
"""
Called when a user leaves a channel.
"""
nick = prefix.split('!')[0] nick = prefix.split('!')[0]
channel = params[0] channel = params[0]
if len(params) >= 2: if len(params) >= 2:
@ -531,19 +525,10 @@ class IRCBot(IRCClient):
self.userLeft(prefix, channel, message) self.userLeft(prefix, channel, message)
def irc_QUIT(self, prefix, params): def irc_QUIT(self, prefix, params):
"""
Called when a user has quit.
"""
nick = prefix.split('!')[0] nick = prefix.split('!')[0]
#if nick == self.nickname:
#self.botQuit(prefix, params[0])
#else:
self.userQuit(prefix, params[0]) self.userQuit(prefix, params[0])
def irc_NICK(self, prefix, params): def irc_NICK(self, prefix, params):
"""
Called when a user changes their nickname.
"""
nick = prefix.split('!', 1)[0] nick = prefix.split('!', 1)[0]
if nick == self.nickname: if nick == self.nickname:
self.nickChanged(prefix, params[0]) self.nickChanged(prefix, params[0])
@ -551,10 +536,6 @@ class IRCBot(IRCClient):
self.userRenamed(prefix, params[0]) self.userRenamed(prefix, params[0])
def irc_KICK(self, prefix, params): def irc_KICK(self, prefix, params):
"""
Called when a user is kicked from a channel.
"""
#kicker = prefix.split('!')[0]
channel = params[0] channel = params[0]
kicked = params[1] kicked = params[1]
message = params[-1] message = params[-1]
@ -562,18 +543,13 @@ class IRCBot(IRCClient):
self.userKicked(kicked, channel, prefix, message) self.userKicked(kicked, channel, prefix, message)
def irc_TOPIC(self, prefix, params): def irc_TOPIC(self, prefix, params):
"""
Someone in the channel set the topic.
"""
#user = prefix.split('!')[0]
channel = params[0] channel = params[0]
newtopic = params[1] newtopic = params[1]
self.topicUpdated(prefix, channel, newtopic) self.topicUpdated(prefix, channel, newtopic)
#END hacks # End of Twisted hackery
def signedOn(self): def signedOn(self):
log("signed on: %s - %i" % (self.net, self.num)) log("signed on: %s - %i" % (self.net, self.num))
#self.event(type="conn", status="connected")
sendRelayNotification({"type": "conn", "net": self.net, "num": self.num, "status": "signedon"}) sendRelayNotification({"type": "conn", "net": self.net, "num": self.num, "status": "signedon"})
def joined(self, channel): def joined(self, channel):
@ -581,8 +557,6 @@ class IRCBot(IRCClient):
self.channels.append(channel) self.channels.append(channel)
self.names(channel).addCallback(self.got_names) self.names(channel).addCallback(self.got_names)
if main.config["Toggles"]["Who"]: if main.config["Toggles"]["Who"]:
#self.who(channel).addCallback(self.got_who)
#self.who(channel)
lc = LoopingCall(self.who, channel) lc = LoopingCall(self.who, channel)
self._getWho[channel] = lc self._getWho[channel] = lc
intrange = main.config["Tweaks"]["Delays"]["WhoRange"] intrange = main.config["Tweaks"]["Delays"]["WhoRange"]

View File

@ -38,8 +38,6 @@ class Server(Protocol):
del main.connections[self.addr] del main.connections[self.addr]
else: else:
warn("Tried to remove a non-existant connection.") warn("Tried to remove a non-existant connection.")
if self.addr in main.MonitorPool:
main.MonitorPool.remove(self.addr)
class ServerFactory(Factory): class ServerFactory(Factory):
def buildProtocol(self, addr): def buildProtocol(self, addr):

23
main.py
View File

@ -6,6 +6,7 @@ from os import urandom
from utils.logging.log import * from utils.logging.log import *
# List of errors ZNC can give us
ZNCErrors = ["Error:", ZNCErrors = ["Error:",
"Unable to load", "Unable to load",
"does not exist", "does not exist",
@ -28,17 +29,32 @@ filemap = {
"network": ["network.dat", "network list", "pickle"] "network": ["network.dat", "network list", "pickle"]
} }
# Connections to the plain-text interface
connections = {} connections = {}
# Connections to the JSON interface
relayConnections = {} relayConnections = {}
# Mapping of network names to Protocol (IRCClient) instances
IRCPool = {} IRCPool = {}
# Mapping of network names to Reactor instances
# Needed for calling .disconnect()
ReactorPool = {} ReactorPool = {}
# Mapping of network names to Factory instances
# Needed for calling .stopTrying()
FactoryPool = {} FactoryPool = {}
# Temporary store for channels allocated after a LIST
# Will get purged as the instances fire up and pop() from
# their respective keys in here
TempChan = {} TempChan = {}
MonitorPool = [] # Mapping of command names to their functions
CommandMap = {} CommandMap = {}
# Incremented by 1 for each event reaching modules.counters.event()
# and cloned into lastMinuteSample every minute
runningSample = 0 runningSample = 0
lastMinuteSample = 0 lastMinuteSample = 0
@ -46,6 +62,7 @@ lastMinuteSample = 0
hashKey = urandom(16) hashKey = urandom(16)
lastEvents = {} lastEvents = {}
# Get networks that are currently online and dedupliate
def liveNets(): def liveNets():
networks = set() networks = set()
for i in IRCPool.keys(): for i in IRCPool.keys():
@ -67,7 +84,7 @@ def loadConf(var):
with open(configPath+filemap[var][0], "r") as f: with open(configPath+filemap[var][0], "r") as f:
globals()[var] = json.load(f) globals()[var] = json.load(f)
if var == "alias": if var == "alias":
# This is a hack to convert all the keys into integers since JSON # This is a workaround to convert all the keys into integers since JSON
# turns them into strings... # turns them into strings...
# Dammit Jason! # Dammit Jason!
global alias global alias