Implement indexing into Apache Druid #1

Closed
m wants to merge 263 commits from druid into master
2 changed files with 66 additions and 25 deletions
Showing only changes of commit c145e5cf18 - Show all commits

View File

@ -8,14 +8,30 @@ import modules.provision
from utils.logging.debug import debug, trace
from utils.logging.log import error, log, warn
def getEnabledRelays(net):
"""
Get a list of enabled relays for a network.
:param net: network
:rtype: list of int
:return: list of enabled relay numbers
"""
enabledRelays = [x for x in main.network[net].relays.keys() if main.network[net].relays[x]["enabled"]]
return enabledRelays
def getActiveRelays(net):
"""
Get a list of active relays for a network.
:param net: network
:rtype: list of int
:return: list of active relay numbers"""
activeRelays = [x for x in main.network[net].relays.keys() if main.network[net].relays[x]["enabled"]]
:return: list of getEnabledRelays relay numbers
"""
enabledRelays = getEnabledRelays(net)
activeRelays = []
for i in enabledRelays:
name = net + str(i)
if name in main.IRCPool.keys():
if main.IRCPool[name].authenticated and main.IRCPool[name].isconnected:
activeRelays.append(i)
return activeRelays
@ -27,27 +43,40 @@ def allRelaysActive(net):
:return: True if all relays are active and authenticated, False otherwise
"""
activeRelays = getActiveRelays(net)
debug(f"allRelaysActive() active relays for {net}: {activeRelays}")
relayNum = len(activeRelays)
existNum = 0
for i in activeRelays:
enabledRelays = getEnabledRelays(net)
relaysActive = len(activeRelays) == len(enabledRelays)
debug(f"allRelaysActive() {net}: {relaysActive} ({activeRelays}/{enabledRelays})")
return relaysActive
def getAverageChanlimit(net):
"""
Get the average channel limit for a network.
:param net: network
:rtype: int
:return: average channel limit
"""
total = 0
for i in getActiveRelays(net):
name = net + str(i)
if name in main.IRCPool.keys():
if main.IRCPool[name].authenticated and main.IRCPool[name].isconnected:
existNum += 1
else:
debug(f"allRelaysActive() {name} is not authenticated or connected")
debug(
(
f"allRelaysActive() {name} auth:{main.IRCPool[name].authenticated} "
f"connected:{main.IRCPool[name].isconnected}"
)
)
debug(f"allRelaysActive() finished, {existNum}/{relayNum} relays active for {net}")
if existNum == relayNum:
return True
return False
total += main.IRCPool[name].chanlimit
avg_chanlimit = total / len(getActiveRelays(net))
debug(f"getAverageChanlimit() {net}: {avg_chanlimit}")
return avg_chanlimit
def getSumChanlimit(net):
"""
Get the sum of all channel limits for a network.
:param net: network
:rtype: int
:return: sum of channel limits
"""
total = 0
for i in getActiveRelays(net):
name = net + str(i)
if name in main.IRCPool.keys():
total += main.IRCPool[name].chanlimit
return total
def getChanFree(net, new):
"""
@ -245,17 +274,27 @@ def keepChannels(net, listinfo, mean, sigrelay, relay, chanlimit):
debug(f"keepChannels() max_chans:{max_chans}")
if coverAll:
needed = relay - len(getActiveRelays(net))
debug(f"keepChannels() coverAll asking to provision {needed} relays for {net} relay:{relay}")
newNums = modules.provision.provisionMultipleRelays(net, needed)
if needed:
debug(f"keepChannels() coverAll asking to provision {needed} relays for {net} relay:{relay}")
newNums = modules.provision.provisionMultipleRelays(net, needed)
else:
newNums = []
flist = [i[0] for i in listinfo]
chosen = sorted(flist, reverse=True, key=lambda x: x[1])[:max_chans]
debug(f"keepChannels() {net}: joining {len(chosen)}/{len(flist)} channels")
trace(f"keepChannels() {net}: joining:{chosen}")
populateChans(net, chosen, newNums)
else:
needed = sigrelay - len(getActiveRelays(net))
debug(f"keepChannels() NOT coverAll asking to provision {needed} relays for {net} sigrelay:{sigrelay}")
newNums = modules.provision.provisionMultipleRelays(net, needed)
if needed:
debug(f"keepChannels() NOT coverAll asking to provision {needed} relays for {net} sigrelay:{sigrelay}")
newNums = modules.provision.provisionMultipleRelays(net, needed)
else:
newNums = []
siglist = [i[0] for i in listinfo if int(i[1]) > mean]
chosen = sorted(siglist, reverse=True, key=lambda x: x[1])[:max_chans]
chosen = sorted(flist, reverse=True, key=lambda x: x[1])[:max_chans]
debug(f"keepChannels() {net}: joining {len(chosen)}/{len(flist)} channels")
trace(f"keepChannels() {net}: joining:{chosen}")
populateChans(net, chosen, newNums)
notifyJoin(net)

View File

@ -75,6 +75,8 @@ def provisionRelay(num, network): # provision user and network data
def provisionMultipleRelays(net, relaysNeeded):
if not relaysNeeded:
return []
if not main.config["ChanKeep"]["Provision"]:
warn(f"Asked to create {relaysNeeded} relays for {net}, but provisioning is disabled")
return []