Implement indexing into Apache Druid #1

Closed
m wants to merge 263 commits from druid into master
3 changed files with 135 additions and 10 deletions
Showing only changes of commit 15583bdaab - Show all commits

View File

@ -5,7 +5,7 @@ from klein import Klein
from twisted.web.server import Request from twisted.web.server import Request
import main import main
from modules import userinfo from modules import chankeep, userinfo
from utils.logging.log import warn from utils.logging.log import warn
@ -195,7 +195,7 @@ class API(object):
@login_required @login_required
def irc_network(self, request, net): def irc_network(self, request, net):
if net not in main.network.keys(): if net not in main.network.keys():
return dumps(False) return dumps({"success": False, "reason": "no such net."})
inst = main.network[net] inst = main.network[net]
network = {} network = {}
network["net"] = inst.net network["net"] = inst.net
@ -217,7 +217,7 @@ class API(object):
except JSONDecodeError: except JSONDecodeError:
return "Invalid JSON" return "Invalid JSON"
if net not in main.network.keys(): if net not in main.network.keys():
return dumps(False) return dumps({"success": False, "reason": "no such net."})
inst = main.network[net] inst = main.network[net]
for item in data: for item in data:
if item == "auth": if item == "auth":
@ -252,7 +252,7 @@ class API(object):
@login_required @login_required
def irc_network_relays(self, request, net): def irc_network_relays(self, request, net):
if net not in main.network.keys(): if net not in main.network.keys():
return dumps(False) return dumps({"success": False, "reason": "no such net."})
relays_inst = main.network[net].relays relays_inst = main.network[net].relays
relays = [] relays = []
for num in relays_inst.keys(): for num in relays_inst.keys():
@ -268,11 +268,37 @@ class API(object):
return dumps({"relays": relays}) return dumps({"relays": relays})
@app.route("/irc/network/<net>/<num>/", methods=["POST"])
@login_required
def irc_network_relay(self, request, net, num):
try:
data = loads(request.content.read())
except JSONDecodeError:
return "Invalid JSON"
if net not in main.network.keys():
return dumps({"success": False, "reason": "no such net."})
if not num.isdigit():
return dumps({"success": False, "reason": "invalid num: not a number."})
num = int(num)
net_inst = main.network[net]
if num not in net_inst.relays:
return dumps({"success": False, "reason": "network has no such relay."})
if "status" in data:
if not type(data["status"]) == int:
return dumps({"success": False, "reason": "invalid type for enabled."})
enabled = data["status"]
if enabled:
net_inst.enable_relay(num)
else:
net_inst.disable_relay(num)
main.saveConf("network")
return dumps({"success": True})
@app.route("/irc/network/<net>/channels/", methods=["POST"]) @app.route("/irc/network/<net>/channels/", methods=["POST"])
@login_required @login_required
def irc_network_channels(self, request, net): def irc_network_channels(self, request, net):
if net not in main.network.keys(): if net not in main.network.keys():
return dumps(False) return dumps({"success": False, "reason": "no such net."})
relays_inst = main.network[net].relays relays_inst = main.network[net].relays
channels = {} channels = {}
for num in relays_inst.keys(): for num in relays_inst.keys():
@ -284,3 +310,34 @@ class API(object):
channels[channel] = channels_annotated[channel] channels[channel] = channels_annotated[channel]
return dumps({"channels": channels}) return dumps({"channels": channels})
@app.route("/irc/network/<net>/channel/<channel>/", methods=["DELETE"])
@login_required
def irc_network_channel_part(self, request, net, channel):
if net not in main.network.keys():
return dumps({"success": False, "reason": "no such net."})
parted = chankeep.partSingle(net, channel)
if not parted:
dumps({"success": False, "reason": "no channels matched."})
return dumps({"success": True, "relays": parted})
@app.route("/irc/network/<net>/channel/<channel>/", methods=["PUT"])
@login_required
def irc_network_channel_join(self, request, net, channel):
if net not in main.network.keys():
return dumps({"success": False, "reason": "no such net."})
joined = chankeep.joinSingle(net, channel)
if not joined:
dumps({"success": False, "reason": "no channels joined."})
return dumps({"success": True, "relays": joined})
@app.route("/aliases/", methods=["GET"])
@login_required
def aliases(self, request):
alias_list = []
for num, alias in main.alias.items():
alias_dup = dict(alias)
alias_dup["num"] = num
alias_list.append(alias_dup)
return dumps({"aliases": alias_list})

View File

@ -23,6 +23,15 @@ def allRelaysActive(net):
def getChanFree(net, new): def getChanFree(net, new):
"""
Get a dictionary with the free channel spaces for
each relay, and a channel limit.
Example return:
({1: 99}, 100)
:param net: network
:param new: list of newly provisioned relays to skip
:return: ({relay: channel spaces}, channel limit)
"""
chanfree = {} chanfree = {}
chanlimits = set() chanlimits = set()
for i in main.network[net].relays.keys(): for i in main.network[net].relays.keys():
@ -123,16 +132,38 @@ def keepChannels(net, listinfo, mean, sigrelay, relay):
def joinSingle(net, channel): def joinSingle(net, channel):
if allRelaysActive(net): if allRelaysActive(net):
chanfree = getChanFree(net, []) # Use the algorithm to allocate our channel to a relay
print("chanfree", chanfree) eca = emptyChanAllocate(net, [channel], None, [])
for i in chanfree[0]: if not len(eca.keys()) == 1:
if chanfree[0][i] < 0: return False
print("JOIN CHAN") num = list(eca.keys())[0]
name = f"{net}{num}"
if name not in main.IRCPool:
return False
main.IRCPool[name].join(channel)
return num
else: else:
error("All relays for %s are not active" % net) error("All relays for %s are not active" % net)
return False return False
def partSingle(net, channel):
"""
Iterate over all the relays of net and part channels matching channel.
:param net:
:param channel:
:return:
"""
parted = []
for i in main.network[net].relays.keys():
name = f"{net}{i}"
if name in main.IRCPool.keys():
if channel in main.IRCPool[name].channels:
main.IRCPool[name].part(channel)
parted.append(str(i))
return parted
def nukeNetwork(net): def nukeNetwork(net):
# purgeRecords(net) # purgeRecords(net)
# p = main.g.pipeline() # p = main.g.pipeline()

View File

@ -6,6 +6,7 @@ from core.bot import IRCBotFactory
from modules import alias from modules import alias
from modules.chankeep import nukeNetwork from modules.chankeep import nukeNetwork
from modules.regproc import needToRegister from modules.regproc import needToRegister
from utils.deliver_relay_commands import deliverRelayCommands
from utils.get import getRelay from utils.get import getRelay
from utils.logging.log import log from utils.logging.log import log
@ -48,6 +49,42 @@ class Network:
# self.start_bot(num) # self.start_bot(num)
return num, main.alias[num]["nick"] return num, main.alias[num]["nick"]
def enable_relay(self, num):
"""
Enable a relay for this network.
Send a command to ZNC to connect.
"""
self.relays[num]["enabled"] = True
user = main.alias[num]["nick"]
commands = {"status": ["Connect"]}
name = f"{self.net}{num}"
deliverRelayCommands(num, commands, user=user + "/" + self.net)
main.saveConf("network")
if name not in main.IRCPool.keys():
self.start_bot(num)
def disable_relay(self, num):
"""
Disable a relay for this network.
Send a command to ZNC to disconnect.
Stop trying to connect to the relay.
"""
self.relays[num]["enabled"] = False
user = main.alias[num]["nick"]
# relay = main.network[spl[1]].relays[relayNum]
commands = {"status": ["Disconnect"]}
name = f"{self.net}{num}"
deliverRelayCommands(num, commands, user=user + "/" + self.net)
main.saveConf("network")
if name in main.ReactorPool.keys():
if name in main.FactoryPool.keys():
main.FactoryPool[name].stopTrying()
main.ReactorPool[name].disconnect()
if name in main.IRCPool.keys():
del main.IRCPool[name]
del main.ReactorPool[name]
del main.FactoryPool[name]
def killAliases(self, aliasList): def killAliases(self, aliasList):
for i in aliasList: for i in aliasList:
name = self.net + str(i) name = self.net + str(i)