Implement indexing into Apache Druid #1

Closed
m wants to merge 263 commits from druid into master
3 changed files with 135 additions and 10 deletions
Showing only changes of commit 15583bdaab - Show all commits

View File

@ -5,7 +5,7 @@ from klein import Klein
from twisted.web.server import Request
import main
from modules import userinfo
from modules import chankeep, userinfo
from utils.logging.log import warn
@ -195,7 +195,7 @@ class API(object):
@login_required
def irc_network(self, request, net):
if net not in main.network.keys():
return dumps(False)
return dumps({"success": False, "reason": "no such net."})
inst = main.network[net]
network = {}
network["net"] = inst.net
@ -217,7 +217,7 @@ class API(object):
except JSONDecodeError:
return "Invalid JSON"
if net not in main.network.keys():
return dumps(False)
return dumps({"success": False, "reason": "no such net."})
inst = main.network[net]
for item in data:
if item == "auth":
@ -252,7 +252,7 @@ class API(object):
@login_required
def irc_network_relays(self, request, net):
if net not in main.network.keys():
return dumps(False)
return dumps({"success": False, "reason": "no such net."})
relays_inst = main.network[net].relays
relays = []
for num in relays_inst.keys():
@ -268,11 +268,37 @@ class API(object):
return dumps({"relays": relays})
@app.route("/irc/network/<net>/<num>/", methods=["POST"])
@login_required
def irc_network_relay(self, request, net, num):
try:
data = loads(request.content.read())
except JSONDecodeError:
return "Invalid JSON"
if net not in main.network.keys():
return dumps({"success": False, "reason": "no such net."})
if not num.isdigit():
return dumps({"success": False, "reason": "invalid num: not a number."})
num = int(num)
net_inst = main.network[net]
if num not in net_inst.relays:
return dumps({"success": False, "reason": "network has no such relay."})
if "status" in data:
if not type(data["status"]) == int:
return dumps({"success": False, "reason": "invalid type for enabled."})
enabled = data["status"]
if enabled:
net_inst.enable_relay(num)
else:
net_inst.disable_relay(num)
main.saveConf("network")
return dumps({"success": True})
@app.route("/irc/network/<net>/channels/", methods=["POST"])
@login_required
def irc_network_channels(self, request, net):
if net not in main.network.keys():
return dumps(False)
return dumps({"success": False, "reason": "no such net."})
relays_inst = main.network[net].relays
channels = {}
for num in relays_inst.keys():
@ -284,3 +310,34 @@ class API(object):
channels[channel] = channels_annotated[channel]
return dumps({"channels": channels})
@app.route("/irc/network/<net>/channel/<channel>/", methods=["DELETE"])
@login_required
def irc_network_channel_part(self, request, net, channel):
if net not in main.network.keys():
return dumps({"success": False, "reason": "no such net."})
parted = chankeep.partSingle(net, channel)
if not parted:
dumps({"success": False, "reason": "no channels matched."})
return dumps({"success": True, "relays": parted})
@app.route("/irc/network/<net>/channel/<channel>/", methods=["PUT"])
@login_required
def irc_network_channel_join(self, request, net, channel):
if net not in main.network.keys():
return dumps({"success": False, "reason": "no such net."})
joined = chankeep.joinSingle(net, channel)
if not joined:
dumps({"success": False, "reason": "no channels joined."})
return dumps({"success": True, "relays": joined})
@app.route("/aliases/", methods=["GET"])
@login_required
def aliases(self, request):
alias_list = []
for num, alias in main.alias.items():
alias_dup = dict(alias)
alias_dup["num"] = num
alias_list.append(alias_dup)
return dumps({"aliases": alias_list})

View File

@ -23,6 +23,15 @@ def allRelaysActive(net):
def getChanFree(net, new):
"""
Get a dictionary with the free channel spaces for
each relay, and a channel limit.
Example return:
({1: 99}, 100)
:param net: network
:param new: list of newly provisioned relays to skip
:return: ({relay: channel spaces}, channel limit)
"""
chanfree = {}
chanlimits = set()
for i in main.network[net].relays.keys():
@ -123,16 +132,38 @@ def keepChannels(net, listinfo, mean, sigrelay, relay):
def joinSingle(net, channel):
if allRelaysActive(net):
chanfree = getChanFree(net, [])
print("chanfree", chanfree)
for i in chanfree[0]:
if chanfree[0][i] < 0:
print("JOIN CHAN")
# Use the algorithm to allocate our channel to a relay
eca = emptyChanAllocate(net, [channel], None, [])
if not len(eca.keys()) == 1:
return False
num = list(eca.keys())[0]
name = f"{net}{num}"
if name not in main.IRCPool:
return False
main.IRCPool[name].join(channel)
return num
else:
error("All relays for %s are not active" % net)
return False
def partSingle(net, channel):
"""
Iterate over all the relays of net and part channels matching channel.
:param net:
:param channel:
:return:
"""
parted = []
for i in main.network[net].relays.keys():
name = f"{net}{i}"
if name in main.IRCPool.keys():
if channel in main.IRCPool[name].channels:
main.IRCPool[name].part(channel)
parted.append(str(i))
return parted
def nukeNetwork(net):
# purgeRecords(net)
# p = main.g.pipeline()

View File

@ -6,6 +6,7 @@ from core.bot import IRCBotFactory
from modules import alias
from modules.chankeep import nukeNetwork
from modules.regproc import needToRegister
from utils.deliver_relay_commands import deliverRelayCommands
from utils.get import getRelay
from utils.logging.log import log
@ -48,6 +49,42 @@ class Network:
# self.start_bot(num)
return num, main.alias[num]["nick"]
def enable_relay(self, num):
"""
Enable a relay for this network.
Send a command to ZNC to connect.
"""
self.relays[num]["enabled"] = True
user = main.alias[num]["nick"]
commands = {"status": ["Connect"]}
name = f"{self.net}{num}"
deliverRelayCommands(num, commands, user=user + "/" + self.net)
main.saveConf("network")
if name not in main.IRCPool.keys():
self.start_bot(num)
def disable_relay(self, num):
"""
Disable a relay for this network.
Send a command to ZNC to disconnect.
Stop trying to connect to the relay.
"""
self.relays[num]["enabled"] = False
user = main.alias[num]["nick"]
# relay = main.network[spl[1]].relays[relayNum]
commands = {"status": ["Disconnect"]}
name = f"{self.net}{num}"
deliverRelayCommands(num, commands, user=user + "/" + self.net)
main.saveConf("network")
if name in main.ReactorPool.keys():
if name in main.FactoryPool.keys():
main.FactoryPool[name].stopTrying()
main.ReactorPool[name].disconnect()
if name in main.IRCPool.keys():
del main.IRCPool[name]
del main.ReactorPool[name]
del main.FactoryPool[name]
def killAliases(self, aliasList):
for i in aliasList:
name = self.net + str(i)