Implement indexing into Apache Druid #1

Closed
m wants to merge 263 commits from druid into master
2 changed files with 42 additions and 15 deletions
Showing only changes of commit 836e621063 - Show all commits

View File

@ -198,7 +198,6 @@ class API(object):
def irc_network(self, request, net): def irc_network(self, request, net):
if net not in main.network.keys(): if net not in main.network.keys():
return dumps({"success": False, "reason": "no such net."}) return dumps({"success": False, "reason": "no such net."})
first_relay = helpers.get_first_relay(net)
inst = main.network[net] inst = main.network[net]
network = {} network = {}
network["net"] = inst.net network["net"] = inst.net
@ -210,9 +209,6 @@ class API(object):
network["relays"] = len(inst.relays) network["relays"] = len(inst.relays)
network["channels"] = userinfo.getTotalChanNum(net) network["channels"] = userinfo.getTotalChanNum(net)
network["records"] = userinfo.getNumWhoEntries(net) network["records"] = userinfo.getNumWhoEntries(net)
if first_relay:
network["chanlimit_live"] = first_relay.chanlimit
network["chanlimit_conf"] = inst.chanlimit
return dumps(network) return dumps(network)
@app.route("/irc/network/<net>/", methods=["DELETE"]) @app.route("/irc/network/<net>/", methods=["DELETE"])
@ -483,6 +479,15 @@ class API(object):
first_relay.list() first_relay.list()
return dumps({"success": True, "message": f"requested list with instance {first_relay.num} of {net}"}) return dumps({"success": True, "message": f"requested list with instance {first_relay.num} of {net}"})
@app.route("/irc/list/<net>/", methods=["GET"])
@login_required
def get_irc_list_info(self, request, net):
if net not in main.network.keys():
return dumps({"success": False, "reason": "no such net."})
listinfo = chankeep.getListInfo(net)
return dumps({"success": True, "listinfo": listinfo})
@app.route("/irc/msg/<net>/<num>/<channel>/", methods=["PUT"]) @app.route("/irc/msg/<net>/<num>/<channel>/", methods=["PUT"])
@login_required @login_required
def irc_send_message(self, request, net, num, channel): def irc_send_message(self, request, net, num, channel):
@ -527,3 +532,4 @@ class API(object):
if name not in main.IRCPool.keys(): if name not in main.IRCPool.keys():
return dumps({"success": False, "reason": f"relay {num} not on {net}"}) return dumps({"success": False, "reason": f"relay {num} not on {net}"})
return dumps({"nickname": main.IRCPool[name].nickname}) return dumps({"nickname": main.IRCPool[name].nickname})

View File

@ -347,21 +347,22 @@ def _initialList(net, num, listinfo, chanlimit):
relay = ceil(listlength / chanlimit) relay = ceil(listlength / chanlimit)
abase = "analytics.list.%s" % net abase = "analytics.list.%s" % net
main.g.delete(abase)
p = main.g.pipeline() p = main.g.pipeline()
# See docstring for meanings # See docstring for meanings
p.hset(abase, "mean", mean) p.hset(abase, "mean", mean)
p.hset(abase, "total", listlength) p.hset(abase, "total_chans", listlength)
p.hset(abase, "sigtotal", siglength) p.hset(abase, "big_chans", siglength)
p.hset(abase, "insigtotal", insiglength) p.hset(abase, "small_chans", insiglength)
p.hset(abase, "sigperc", round(siglength / listlength * 100, 2)) p.hset(abase, "big_chan_perc", round(siglength / listlength * 100, 2))
p.hset(abase, "insigperc", round(insiglength / listlength * 100, 2)) p.hset(abase, "small_chan_perc", round(insiglength / listlength * 100, 2))
p.hset(abase, "cumul", cumul) p.hset(abase, "total_cumul_mem", cumul)
p.hset(abase, "sigcumul", sigcumul) p.hset(abase, "big_chan_cumul_mem", sigcumul)
p.hset(abase, "insigcumul", insigcumul) p.hset(abase, "small_chan_cumul_mem", insigcumul)
p.hset(abase, "relay", relay) p.hset(abase, "relays_for_all_chans", relay)
p.hset(abase, "sigrelay", sigrelay) p.hset(abase, "relays_for_big_chans", sigrelay)
p.hset(abase, "insigrelay", ceil(insiglength / chanlimit)) p.hset(abase, "relays_for_small_chans", ceil(insiglength / chanlimit))
debug( debug(
( (
f"_initialList() net:{net} num:{num} listlength:{listlength} " f"_initialList() net:{net} num:{num} listlength:{listlength} "
@ -383,6 +384,26 @@ def _initialList(net, num, listinfo, chanlimit):
# return (listinfo, mean, sigrelay, relay) # return (listinfo, mean, sigrelay, relay)
def convert(data):
"""
Recursively convert a dictionary.
"""
if isinstance(data, bytes):
return data.decode("ascii")
if isinstance(data, dict):
return dict(map(convert, data.items()))
if isinstance(data, tuple):
return map(convert, data)
if isinstance(data, list):
return list(map(convert, data))
return data
def getListInfo(net):
abase = f"analytics.list.{net}"
info = main.g.hgetall(abase)
return convert(info)
def initialList(net, num, listinfo, chanlimit): def initialList(net, num, listinfo, chanlimit):
""" """