Implement indexing into Apache Druid #1

Closed
m wants to merge 263 commits from druid into master
6 changed files with 55 additions and 16 deletions
Showing only changes of commit edfb3f15eb - Show all commits

View File

@ -198,6 +198,7 @@ class API(object):
def irc_network(self, request, net): def irc_network(self, request, net):
if net not in main.network.keys(): if net not in main.network.keys():
return dumps({"success": False, "reason": "no such net."}) return dumps({"success": False, "reason": "no such net."})
first_relay = helpers.get_first_relay(net)
inst = main.network[net] inst = main.network[net]
network = {} network = {}
network["net"] = inst.net network["net"] = inst.net
@ -209,6 +210,9 @@ class API(object):
network["relays"] = len(inst.relays) network["relays"] = len(inst.relays)
network["channels"] = userinfo.getTotalChanNum(net) network["channels"] = userinfo.getTotalChanNum(net)
network["records"] = userinfo.getNumWhoEntries(net) network["records"] = userinfo.getNumWhoEntries(net)
if first_relay:
network["chanlimit_live"] = first_relay.chanlimit
network["chanlimit_conf"] = inst.chanlimit
return dumps(network) return dumps(network)
@app.route("/irc/network/<net>/", methods=["DELETE"]) @app.route("/irc/network/<net>/", methods=["DELETE"])

View File

@ -60,6 +60,7 @@ def getChanFree(net, new):
return False return False
return (chanfree, chanlimits.pop()) return (chanfree, chanlimits.pop())
def getTotalChans(net): def getTotalChans(net):
total = 0 total = 0
for i in getActiveRelays(net): for i in getActiveRelays(net):

View File

@ -1,3 +1,5 @@
from copy import deepcopy
from twisted.internet import reactor from twisted.internet import reactor
from twisted.internet.ssl import DefaultOpenSSLContextFactory from twisted.internet.ssl import DefaultOpenSSLContextFactory
@ -11,6 +13,31 @@ from utils.get import getRelay
from utils.logging.log import log from utils.logging.log import log
def migrate():
existing = deepcopy(main.network)
log("Migrating network configuration")
log(f"Existing network configuration: {existing}")
for net, net_inst in existing.items():
log(f"Migrating network {net}")
net = net_inst.net
host = net_inst.host
port = net_inst.port
security = net_inst.security
auth = net_inst.auth
last = net_inst.last
relays = net_inst.relays
aliases = net_inst.aliases
new_net = Network(net, host, port, security, auth)
log(f"New network for {net}: {new_net}")
new_net.last = last
new_net.relays = relays
new_net.aliases = aliases
main.network[net] = new_net
main.saveConf("network")
log("Finished migrating network configuration")
class Network: class Network:
def __init__(self, net, host, port, security, auth): def __init__(self, net, host, port, security, auth):
self.net = net self.net = net
@ -18,6 +45,7 @@ class Network:
self.port = port self.port = port
self.security = security self.security = security
self.auth = auth self.auth = auth
self.chanlimit = None
self.last = 1 self.last = 1
self.relays = {} self.relays = {}

View File

@ -9,11 +9,6 @@ from utils.logging.log import warn
def provisionUserNetworkData( def provisionUserNetworkData(
num, nick, altnick, ident, realname, emails, network, host, port, security, auth, password num, nick, altnick, ident, realname, emails, network, host, port, security, auth, password
): ):
print("nick", nick)
print("altnick", altnick)
print("emails", emails)
print("ident", ident)
print("realname", realname)
commands = {} commands = {}
stage2commands = {} stage2commands = {}
stage2commands["status"] = [] stage2commands["status"] = []

View File

@ -1,10 +1,10 @@
import logging from math import ceil
from random import randint
from unittest import TestCase from unittest import TestCase
from unittest.mock import MagicMock, patch from unittest.mock import MagicMock, patch
from random import randint
from modules import chankeep from modules import chankeep
from math import ceil
import heapq
class TestChanKeep(TestCase): class TestChanKeep(TestCase):
def setUp(self): def setUp(self):
@ -48,7 +48,7 @@ class TestChanKeep(TestCase):
chanlimit = 5 chanlimit = 5
max_chans = instances * chanlimit max_chans = instances * chanlimit
listinfo = self.generate_listinfo(ranges=[[1000, 1, 2], [200, 400, 800], [10, 1000, 2000]]) listinfo = self.generate_listinfo(ranges=[[1000, 1, 2], [200, 400, 800], [10, 1000, 2000]])
listinfo_num = [x[1] for x in listinfo] # listinfo_num = [x[1] for x in listinfo]
listlength = len(listinfo) listlength = len(listinfo)
cumul = 0 cumul = 0
@ -71,10 +71,17 @@ class TestChanKeep(TestCase):
sigrelay = ceil(siglength / chanlimit) sigrelay = ceil(siglength / chanlimit)
relay = ceil(listlength / chanlimit) relay = ceil(listlength / chanlimit)
print(f"len:{listlength} cumul:{cumul} mean:{mean} siglength:{siglength} insiglength:{insiglength} sigrelay:{sigrelay} relay:{relay} sigcumul:{sigcumul} insigcumul:{insigcumul}") print(
(
f"len:{listlength} cumul:{cumul} mean:{mean} "
f"siglength:{siglength} insiglength:{insiglength} "
f"sigrelay:{sigrelay} relay:{relay} sigcumul:{sigcumul} "
f"insigcumul:{insigcumul}"
)
)
# We want a return between 1000 and 1100 # We want a return between 1000 and 1100
list_insig = [x for x in listinfo_num if x < mean] # list_insig = [x for x in listinfo_num if x < mean]
list_sig = [x for x in listinfo if x[1] > mean] list_sig = [x for x in listinfo if x[1] > mean]
chosen = sorted(list_sig, reverse=True, key=lambda x: x[1])[:max_chans] chosen = sorted(list_sig, reverse=True, key=lambda x: x[1])[:max_chans]
print("CHOSEN", chosen) print("CHOSEN", chosen)
@ -91,7 +98,7 @@ class TestChanKeep(TestCase):
@patch("modules.chankeep.getChanFree") @patch("modules.chankeep.getChanFree")
def test_empty_chan_allocate(self, getchanfree): def test_empty_chan_allocate(self, getchanfree):
getchanfree.return_value = ({1: 600}, 600) # pretend we have 600 channels free getchanfree.return_value = ({1: 600}, 600) # pretend we have 600 channels free
eca = chankeep.emptyChanAllocate(self.net, self.chan_name_list, []) eca = chankeep.emptyChanAllocate(self.net, self.chan_name_list, [])
self.assertEqual(len(eca), 1) self.assertEqual(len(eca), 1)
num = list(eca.keys())[0] num = list(eca.keys())[0]
@ -106,4 +113,4 @@ class TestChanKeep(TestCase):
chans = eca[list(eca.keys())[0]] chans = eca[list(eca.keys())[0]]
self.assertEqual(num, self.num) self.assertEqual(num, self.num)
self.assertEqual(len(chans), 100) self.assertEqual(len(chans), 100)
#self.assertCountEqual(chans, self.chan_name_list) # self.assertCountEqual(chans, self.chan_name_list)

View File

@ -29,7 +29,11 @@ if "--debug" in sys.argv: # yes really
main.config["Debug"] = True main.config["Debug"] = True
if "--trace" in sys.argv: if "--trace" in sys.argv:
main.config["Trace"] = True main.config["Trace"] = True
if "--migrate" in sys.argv:
from modules.network import migrate
migrate()
exit()
loadCommands() loadCommands()