Implement indexing into Apache Druid #1

Closed
m wants to merge 263 commits from druid into master
6 changed files with 55 additions and 16 deletions
Showing only changes of commit edfb3f15eb - Show all commits

View File

@ -198,6 +198,7 @@ class API(object):
def irc_network(self, request, net):
if net not in main.network.keys():
return dumps({"success": False, "reason": "no such net."})
first_relay = helpers.get_first_relay(net)
inst = main.network[net]
network = {}
network["net"] = inst.net
@ -209,6 +210,9 @@ class API(object):
network["relays"] = len(inst.relays)
network["channels"] = userinfo.getTotalChanNum(net)
network["records"] = userinfo.getNumWhoEntries(net)
if first_relay:
network["chanlimit_live"] = first_relay.chanlimit
network["chanlimit_conf"] = inst.chanlimit
return dumps(network)
@app.route("/irc/network/<net>/", methods=["DELETE"])

View File

@ -60,6 +60,7 @@ def getChanFree(net, new):
return False
return (chanfree, chanlimits.pop())
def getTotalChans(net):
total = 0
for i in getActiveRelays(net):

View File

@ -1,3 +1,5 @@
from copy import deepcopy
from twisted.internet import reactor
from twisted.internet.ssl import DefaultOpenSSLContextFactory
@ -11,6 +13,31 @@ from utils.get import getRelay
from utils.logging.log import log
def migrate():
existing = deepcopy(main.network)
log("Migrating network configuration")
log(f"Existing network configuration: {existing}")
for net, net_inst in existing.items():
log(f"Migrating network {net}")
net = net_inst.net
host = net_inst.host
port = net_inst.port
security = net_inst.security
auth = net_inst.auth
last = net_inst.last
relays = net_inst.relays
aliases = net_inst.aliases
new_net = Network(net, host, port, security, auth)
log(f"New network for {net}: {new_net}")
new_net.last = last
new_net.relays = relays
new_net.aliases = aliases
main.network[net] = new_net
main.saveConf("network")
log("Finished migrating network configuration")
class Network:
def __init__(self, net, host, port, security, auth):
self.net = net
@ -18,6 +45,7 @@ class Network:
self.port = port
self.security = security
self.auth = auth
self.chanlimit = None
self.last = 1
self.relays = {}

View File

@ -9,11 +9,6 @@ from utils.logging.log import warn
def provisionUserNetworkData(
num, nick, altnick, ident, realname, emails, network, host, port, security, auth, password
):
print("nick", nick)
print("altnick", altnick)
print("emails", emails)
print("ident", ident)
print("realname", realname)
commands = {}
stage2commands = {}
stage2commands["status"] = []

View File

@ -1,10 +1,10 @@
import logging
from math import ceil
from random import randint
from unittest import TestCase
from unittest.mock import MagicMock, patch
from random import randint
from modules import chankeep
from math import ceil
import heapq
class TestChanKeep(TestCase):
def setUp(self):
@ -48,7 +48,7 @@ class TestChanKeep(TestCase):
chanlimit = 5
max_chans = instances * chanlimit
listinfo = self.generate_listinfo(ranges=[[1000, 1, 2], [200, 400, 800], [10, 1000, 2000]])
listinfo_num = [x[1] for x in listinfo]
# listinfo_num = [x[1] for x in listinfo]
listlength = len(listinfo)
cumul = 0
@ -71,10 +71,17 @@ class TestChanKeep(TestCase):
sigrelay = ceil(siglength / chanlimit)
relay = ceil(listlength / chanlimit)
print(f"len:{listlength} cumul:{cumul} mean:{mean} siglength:{siglength} insiglength:{insiglength} sigrelay:{sigrelay} relay:{relay} sigcumul:{sigcumul} insigcumul:{insigcumul}")
print(
(
f"len:{listlength} cumul:{cumul} mean:{mean} "
f"siglength:{siglength} insiglength:{insiglength} "
f"sigrelay:{sigrelay} relay:{relay} sigcumul:{sigcumul} "
f"insigcumul:{insigcumul}"
)
)
# We want a return between 1000 and 1100
list_insig = [x for x in listinfo_num if x < mean]
# list_insig = [x for x in listinfo_num if x < mean]
list_sig = [x for x in listinfo if x[1] > mean]
chosen = sorted(list_sig, reverse=True, key=lambda x: x[1])[:max_chans]
print("CHOSEN", chosen)

View File

@ -29,7 +29,11 @@ if "--debug" in sys.argv: # yes really
main.config["Debug"] = True
if "--trace" in sys.argv:
main.config["Trace"] = True
if "--migrate" in sys.argv:
from modules.network import migrate
migrate()
exit()
loadCommands()