Implement indexing into Apache Druid #1

Closed
m wants to merge 263 commits from druid into master
1 changed files with 33 additions and 22 deletions
Showing only changes of commit 7415ca5556 - Show all commits

View File

@ -138,8 +138,9 @@ def emptyChanAllocate(net, flist):
allocated = {}
newlist = list(flist)
sum_free = sum(chanfree.values()) # 250
trunc_list = newlist[:sum_free]
chan_slots_used = getTotalChans(net)
max_chans = getSumChanlimit(net) - chan_slots_used
trunc_list = newlist[:max_chans]
debug(f"emptyChanAllocate() {net}: newlist:{len(newlist)} trunc_list:{len(trunc_list)}")
for i in chanfree.keys():
@ -162,6 +163,7 @@ def populateChans(net, clist):
:param new: list of newly provisioned relays to account for"""
# divided = array_split(clist, relay)
allocated = emptyChanAllocate(net, clist)
trace(f"populateChans() allocated:{allocated}")
if not allocated:
return
for i in allocated.keys():
@ -169,6 +171,7 @@ def populateChans(net, clist):
main.TempChan[net][i] = allocated[i]
else:
main.TempChan[net] = {i: allocated[i]}
trace(f"populateChans() TempChan {net}{i}: {allocated[i]}")
def notifyJoin(net):
@ -180,10 +183,11 @@ def notifyJoin(net):
for i in getActiveRelays(net):
name = net + str(i)
if name in main.IRCPool.keys():
trace(f"notifyJoin() {name}")
main.IRCPool[name].checkChannels()
def minifyChans(net, listinfo):
def minifyChans(net, listinfo, as_list=False):
"""
Remove channels from listinfo that are already covered by a relay.
:param net: network
@ -192,18 +196,31 @@ def minifyChans(net, listinfo):
:return: list of channels with joined channels removed
:rtype: list of [channel, num_users]
"""
# We want to make this reusable for joining a bunch of channels.
if as_list:
channel_list = listinfo
if not allRelaysActive(net):
error("All relays for %s are not active, cannot minify list" % net)
return False
for i in getActiveRelays(net):
name = net + str(i)
for x in main.IRCPool[name].channels:
if as_list:
for y in channel_list:
if y == x:
channel_list.remove(y)
else:
for y in listinfo:
if y[0] == x:
listinfo.remove(y)
if not as_list:
if not listinfo:
log("We're on all the channels we want to be on, dropping LIST")
return False
if as_list:
return channel_list
else:
return listinfo
@ -278,19 +295,13 @@ def joinSingle(net, channel):
"""
if "," in channel:
channels = channel.split(",")
eca = emptyChanAllocate(net, channels)
channels = minifyChans(net, channels, as_list=True)
else:
eca = emptyChanAllocate(net, [channel])
if not eca:
return False
if not len(eca.keys()) == 1:
return False
num = list(eca.keys())[0]
name = f"{net}{num}"
if name not in main.IRCPool:
return False
main.IRCPool[name].join(channel)
return num
channels = [channel]
populateChans(net, channels)
notifyJoin(net)
return True
def partSingle(net, channel):