Implement indexing into Apache Druid #1
14
api/views.py
14
api/views.py
|
@ -6,7 +6,7 @@ from klein import Klein
|
|||
from twisted.web.server import Request
|
||||
|
||||
import main
|
||||
from modules import chankeep, provision, userinfo
|
||||
from modules import chankeep, helpers, provision, userinfo
|
||||
from modules.network import Network
|
||||
from utils.logging.log import warn
|
||||
|
||||
|
@ -473,10 +473,8 @@ class API(object):
|
|||
def irc_list_network(self, request, net):
|
||||
if net not in main.network.keys():
|
||||
return dumps({"success": False, "reason": "no such net."})
|
||||
if 1 not in main.network[net].relays.keys():
|
||||
return dumps({"success": False, "reason": f"no first relay on {net}"})
|
||||
name = f"{net}1"
|
||||
if name not in main.IRCPool.keys():
|
||||
return dumps({"success": False, "reason": f"first relay not active for {net}"})
|
||||
main.IRCPool[name].list()
|
||||
return dumps({"success": True, "message": f"requested list with first instance of {net}"})
|
||||
first_relay = helpers.get_first_relay(net)
|
||||
if not first_relay:
|
||||
return dumps({"success": False, "reason": f"could not get first relay for {net}"})
|
||||
first_relay.list()
|
||||
return dumps({"success": True, "message": f"requested list with instance {first_relay.num} of {net}"})
|
||||
|
|
|
@ -1,4 +1,5 @@
|
|||
import main
|
||||
from modules import helpers
|
||||
|
||||
|
||||
class ListCommand:
|
||||
|
@ -9,27 +10,24 @@ class ListCommand:
|
|||
if authed:
|
||||
if length == 1:
|
||||
for i in main.network.keys():
|
||||
if 1 not in main.network[i].relays.keys():
|
||||
first_relay = helpers.get_first_relay(i)
|
||||
####
|
||||
if not first_relay:
|
||||
info("Network has no first instance: %s" % i)
|
||||
continue
|
||||
if not i + "1" in main.IRCPool.keys():
|
||||
info("No IRC instance: %s - 1" % i)
|
||||
continue
|
||||
main.IRCPool[i + "1"].list()
|
||||
success("Requested list with first instance of %s" % i)
|
||||
first_relay.list()
|
||||
success(f"Requested list with instance {first_relay.num} of {i}")
|
||||
return
|
||||
elif length == 2:
|
||||
if not spl[1] in main.network.keys():
|
||||
failure("No such network: %s" % spl[1])
|
||||
return
|
||||
if 1 not in main.network[spl[1]].relays.keys():
|
||||
failure("Network has no first instance")
|
||||
first_relay = helpers.get_first_relay(spl[1])
|
||||
if not first_relay:
|
||||
failure("Could not get first instance")
|
||||
return
|
||||
if spl[1] + "1" not in main.IRCPool.keys():
|
||||
failure("No IRC instance: %s - 1" % spl[1])
|
||||
return
|
||||
main.IRCPool[spl[1] + "1"].list()
|
||||
success("Requested list with first instance of %s" % spl[1])
|
||||
first_relay.list()
|
||||
success(f"Requested list with instance {first_relay.num} of {spl[1]}")
|
||||
return
|
||||
else:
|
||||
incUsage("list")
|
||||
|
|
17
core/bot.py
17
core/bot.py
|
@ -16,7 +16,7 @@ from twisted.words.protocols.irc import (
|
|||
|
||||
import main
|
||||
from core.relay import sendRelayNotification
|
||||
from modules import chankeep, counters, monitor, regproc, userinfo
|
||||
from modules import chankeep, counters, helpers, monitor, regproc, userinfo
|
||||
from utils.dedup import dedup
|
||||
from utils.logging.debug import debug
|
||||
from utils.logging.log import error, log, warn
|
||||
|
@ -458,11 +458,16 @@ class IRCBot(IRCClient):
|
|||
def recheckList(self):
|
||||
allRelays = chankeep.allRelaysActive(self.net)
|
||||
if allRelays:
|
||||
name = self.net + "1"
|
||||
if main.IRCPool[name].wantList is True:
|
||||
main.IRCPool[name].list(nocheck=True)
|
||||
debug("Asking for a list for %s after final relay %i connected" % (self.net, self.num))
|
||||
if self.num == 1: # Only one instance should do a list
|
||||
debug(f"All relays active for {self.net}")
|
||||
first_relay = helpers.get_first_relay(self.net)
|
||||
if first_relay:
|
||||
if first_relay.wantList is True:
|
||||
first_relay.list(nocheck=True)
|
||||
debug("Asking for a list for %s after final relay %i connected" % (self.net, self.num))
|
||||
# name = self.net + "1"
|
||||
|
||||
# if self.num == 1: # Only one instance should do a list
|
||||
if helpers.is_first_relay(self.net, self.num):
|
||||
if self.chanlimit:
|
||||
if allRelays:
|
||||
self.list()
|
||||
|
|
|
@ -9,10 +9,19 @@ from utils.logging.debug import debug
|
|||
from utils.logging.log import error, log, warn
|
||||
|
||||
|
||||
def getActiveRelays(net):
|
||||
activeRelays = [x for x in main.network[net].relays.keys() if main.network[net].relays[x]["enabled"]]
|
||||
return activeRelays
|
||||
|
||||
|
||||
def allRelaysActive(net):
|
||||
relayNum = len(main.network[net].relays.keys())
|
||||
"""
|
||||
Check if all enabled relays are active and authenticated.
|
||||
"""
|
||||
activeRelays = getActiveRelays(net)
|
||||
relayNum = len(activeRelays) + 1
|
||||
existNum = 0
|
||||
for i in main.network[net].relays.keys():
|
||||
for i in activeRelays:
|
||||
name = net + str(i)
|
||||
if name in main.IRCPool.keys():
|
||||
if main.IRCPool[name].authenticated:
|
||||
|
@ -34,7 +43,7 @@ def getChanFree(net, new):
|
|||
"""
|
||||
chanfree = {}
|
||||
chanlimits = set()
|
||||
for i in main.network[net].relays.keys():
|
||||
for i in getActiveRelays(net):
|
||||
if i in new:
|
||||
continue
|
||||
name = net + str(i)
|
||||
|
@ -60,9 +69,8 @@ def emptyChanAllocate(net, flist, relay, new):
|
|||
toalloc = len(flist)
|
||||
if toalloc > sum(chanfree[0].values()):
|
||||
correction = round(toalloc - sum(chanfree[0].values()) / chanfree[1])
|
||||
# print("correction", correction)
|
||||
warn("Ran out of channel spaces, provisioning additional %i relays for %s" % (correction, net))
|
||||
# newNums = modules.provision.provisionMultipleRelays(net, correction)
|
||||
modules.provision.provisionMultipleRelays(net, correction)
|
||||
return False
|
||||
for i in chanfree[0].keys():
|
||||
for x in range(chanfree[0][i]):
|
||||
|
@ -88,7 +96,7 @@ def populateChans(net, clist, relay, new):
|
|||
|
||||
|
||||
def notifyJoin(net):
|
||||
for i in main.network[net].relays.keys():
|
||||
for i in getActiveRelays(net):
|
||||
name = net + str(i)
|
||||
if name in main.IRCPool.keys():
|
||||
main.IRCPool[name].checkChannels()
|
||||
|
@ -98,7 +106,7 @@ def minifyChans(net, listinfo):
|
|||
if not allRelaysActive(net):
|
||||
error("All relays for %s are not active, cannot minify list" % net)
|
||||
return False
|
||||
for i in main.network[net].relays.keys():
|
||||
for i in getActiveRelays(net):
|
||||
name = net + str(i)
|
||||
for x in main.IRCPool[name].channels:
|
||||
for y in listinfo:
|
||||
|
@ -122,12 +130,12 @@ def keepChannels(net, listinfo, mean, sigrelay, relay):
|
|||
error("Network %s is too big to cover: %i relays required" % (net, sigrelay))
|
||||
return
|
||||
if coverAll:
|
||||
needed = relay - len(main.network[net].relays.keys())
|
||||
needed = relay - len(getActiveRelays(net))
|
||||
newNums = modules.provision.provisionMultipleRelays(net, needed)
|
||||
flist = [i[0] for i in listinfo]
|
||||
populateChans(net, flist, relay, newNums)
|
||||
else:
|
||||
needed = sigrelay - len(main.network[net].relays.keys())
|
||||
needed = sigrelay - len(getActiveRelays(net))
|
||||
newNums = modules.provision.provisionMultipleRelays(net, needed)
|
||||
siglist = [i[0] for i in listinfo if int(i[1]) > mean]
|
||||
populateChans(net, siglist, sigrelay, newNums)
|
||||
|
@ -156,7 +164,7 @@ def partSingle(net, channel):
|
|||
:return:
|
||||
"""
|
||||
parted = []
|
||||
for i in main.network[net].relays.keys():
|
||||
for i in getActiveRelays(net):
|
||||
name = f"{net}{i}"
|
||||
if name in main.IRCPool.keys():
|
||||
if channel in main.IRCPool[name].channels:
|
||||
|
|
|
@ -0,0 +1,35 @@
|
|||
import main
|
||||
from modules import chankeep
|
||||
|
||||
def get_first_relay(net):
|
||||
"""
|
||||
Get the first relay in the network.
|
||||
:param net: the network
|
||||
:param num: number or relay
|
||||
:return: IRCPool instance for the IRC bot
|
||||
"""
|
||||
cur_relay = 0
|
||||
max_relay = len(main.network[net].relays.keys())+1
|
||||
activeRelays = chankeep.getActiveRelays(net)
|
||||
while cur_relay != max_relay:
|
||||
cur_relay += 1
|
||||
if cur_relay not in activeRelays:
|
||||
continue
|
||||
name = net + str(cur_relay)
|
||||
if name in main.IRCPool.keys():
|
||||
return main.IRCPool[name]
|
||||
return None
|
||||
|
||||
def is_first_relay(net, num):
|
||||
"""
|
||||
Determine if we are the first relay for the network.
|
||||
:param net: the network
|
||||
:param num: number or relay
|
||||
:return: True if we are the first relay, False otherwise
|
||||
"""
|
||||
cur_relay = 0
|
||||
max_relay = len(main.network[net].relays.keys())
|
||||
while cur_relay > max_relay:
|
||||
name = net + str(cur_relay)
|
||||
if name in main.IRCPool.keys():
|
||||
return cur_relay == num
|
|
@ -3,6 +3,7 @@ from twisted.internet import reactor
|
|||
import main
|
||||
import modules.regproc
|
||||
from utils.deliver_relay_commands import deliverRelayCommands
|
||||
from utils.logging.log import warn
|
||||
|
||||
|
||||
def provisionUserNetworkData(
|
||||
|
@ -72,13 +73,16 @@ def provisionRelay(num, network): # provision user and network data
|
|||
main.network[network].port,
|
||||
main.network[network].security,
|
||||
main.network[network].auth,
|
||||
main.network[network].aliases[num]["password"]
|
||||
main.network[network].aliases[num]["password"],
|
||||
)
|
||||
if main.config["ConnectOnCreate"]:
|
||||
reactor.callLater(10, main.network[network].start_bot, num)
|
||||
|
||||
|
||||
def provisionMultipleRelays(net, relaysNeeded):
|
||||
if not main.config["ChanKeep"]["Provision"]:
|
||||
warn(f"Asked to create {relaysNeeded} relays for {net}, but provisioning is disabled")
|
||||
return 0
|
||||
numsProvisioned = []
|
||||
for i in range(relaysNeeded):
|
||||
num, alias = main.network[net].add_relay()
|
||||
|
|
|
@ -57,8 +57,6 @@ api_enabled = getenv("THRESHOLD_API_ENABLED", main.config["API"]["Enabled"]) in
|
|||
api_address = getenv("THRESHOLD_API_HOST", main.config["API"]["Address"])
|
||||
api_port = int(getenv("THRESHOLD_API_PORT", main.config["API"]["Port"]))
|
||||
|
||||
print("KEY", main.certPath + main.config["Key"])
|
||||
print("CERT", main.certPath + main.config["Certificate"])
|
||||
if __name__ == "__main__":
|
||||
listener = ServerFactory()
|
||||
if listener_ssl is True:
|
||||
|
|
Loading…
Reference in New Issue