Implement indexing into Apache Druid #1

Closed
m wants to merge 263 commits from druid into master
1 changed files with 33 additions and 22 deletions
Showing only changes of commit 7415ca5556 - Show all commits

View File

@ -138,8 +138,9 @@ def emptyChanAllocate(net, flist):
allocated = {} allocated = {}
newlist = list(flist) newlist = list(flist)
sum_free = sum(chanfree.values()) # 250 chan_slots_used = getTotalChans(net)
trunc_list = newlist[:sum_free] max_chans = getSumChanlimit(net) - chan_slots_used
trunc_list = newlist[:max_chans]
debug(f"emptyChanAllocate() {net}: newlist:{len(newlist)} trunc_list:{len(trunc_list)}") debug(f"emptyChanAllocate() {net}: newlist:{len(newlist)} trunc_list:{len(trunc_list)}")
for i in chanfree.keys(): for i in chanfree.keys():
@ -162,6 +163,7 @@ def populateChans(net, clist):
:param new: list of newly provisioned relays to account for""" :param new: list of newly provisioned relays to account for"""
# divided = array_split(clist, relay) # divided = array_split(clist, relay)
allocated = emptyChanAllocate(net, clist) allocated = emptyChanAllocate(net, clist)
trace(f"populateChans() allocated:{allocated}")
if not allocated: if not allocated:
return return
for i in allocated.keys(): for i in allocated.keys():
@ -169,6 +171,7 @@ def populateChans(net, clist):
main.TempChan[net][i] = allocated[i] main.TempChan[net][i] = allocated[i]
else: else:
main.TempChan[net] = {i: allocated[i]} main.TempChan[net] = {i: allocated[i]}
trace(f"populateChans() TempChan {net}{i}: {allocated[i]}")
def notifyJoin(net): def notifyJoin(net):
@ -180,10 +183,11 @@ def notifyJoin(net):
for i in getActiveRelays(net): for i in getActiveRelays(net):
name = net + str(i) name = net + str(i)
if name in main.IRCPool.keys(): if name in main.IRCPool.keys():
trace(f"notifyJoin() {name}")
main.IRCPool[name].checkChannels() main.IRCPool[name].checkChannels()
def minifyChans(net, listinfo): def minifyChans(net, listinfo, as_list=False):
""" """
Remove channels from listinfo that are already covered by a relay. Remove channels from listinfo that are already covered by a relay.
:param net: network :param net: network
@ -192,18 +196,31 @@ def minifyChans(net, listinfo):
:return: list of channels with joined channels removed :return: list of channels with joined channels removed
:rtype: list of [channel, num_users] :rtype: list of [channel, num_users]
""" """
# We want to make this reusable for joining a bunch of channels.
if as_list:
channel_list = listinfo
if not allRelaysActive(net): if not allRelaysActive(net):
error("All relays for %s are not active, cannot minify list" % net) error("All relays for %s are not active, cannot minify list" % net)
return False return False
for i in getActiveRelays(net): for i in getActiveRelays(net):
name = net + str(i) name = net + str(i)
for x in main.IRCPool[name].channels: for x in main.IRCPool[name].channels:
if as_list:
for y in channel_list:
if y == x:
channel_list.remove(y)
else:
for y in listinfo: for y in listinfo:
if y[0] == x: if y[0] == x:
listinfo.remove(y) listinfo.remove(y)
if not as_list:
if not listinfo: if not listinfo:
log("We're on all the channels we want to be on, dropping LIST") log("We're on all the channels we want to be on, dropping LIST")
return False return False
if as_list:
return channel_list
else:
return listinfo return listinfo
@ -278,19 +295,13 @@ def joinSingle(net, channel):
""" """
if "," in channel: if "," in channel:
channels = channel.split(",") channels = channel.split(",")
eca = emptyChanAllocate(net, channels) channels = minifyChans(net, channels, as_list=True)
else: else:
eca = emptyChanAllocate(net, [channel]) channels = [channel]
if not eca: populateChans(net, channels)
return False notifyJoin(net)
if not len(eca.keys()) == 1: return True
return False
num = list(eca.keys())[0]
name = f"{net}{num}"
if name not in main.IRCPool:
return False
main.IRCPool[name].join(channel)
return num
def partSingle(net, channel): def partSingle(net, channel):