Implement indexing into Apache Druid #1

Closed
m wants to merge 263 commits from druid into master
4 changed files with 120 additions and 3 deletions
Showing only changes of commit 5cc38da00e - Show all commits

24
commands/dedup.py Normal file
View File

@ -0,0 +1,24 @@
import main
from modules import chankeep
from modules import helpers
from json import dumps
class DedupCommand:
def __init__(self, *args):
self.dedup(*args)
def dedup(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
if authed:
if length == 1:
dupes = chankeep.getDuplicateChannels()
chankeep.partChannels(dupes)
info(dumps(dupes))
return
elif length == 2:
if spl[1] not in main.network.keys():
failure("No such network: %s" % spl[1])
return
dupes = chankeep.getDuplicateChannels(spl[1])
chankeep.partChannels(dupes)
info(dumps(dupes))
return

View File

@ -628,6 +628,8 @@ class IRCBot(IRCClient):
# End of Twisted hackery # End of Twisted hackery
def regPing(self, negativepass=None, reset=True): def regPing(self, negativepass=None, reset=True):
if not main.config["AutoReg"]:
return
if self.authenticated: if self.authenticated:
return return
if not regproc.needToAuth(self.net): if not regproc.needToAuth(self.net):

View File

@ -4,10 +4,90 @@ from math import ceil
from twisted.internet.threads import deferToThread from twisted.internet.threads import deferToThread
import main import main
from modules import helpers
from utils.logging.debug import debug, trace from utils.logging.debug import debug, trace
from utils.logging.log import error, log, warn from utils.logging.log import error, log, warn
def getAllChannels(net=None):
"""
Get a list of all channels on all relays.
:return: list of channels
"""
channels = {}
if not net:
nets = main.network.keys()
else:
nets = [net]
for net in nets:
relays = helpers.get_connected_relays(net)
for relay in relays:
if net not in channels:
channels[net] = {}
if relay.num not in channels[net]:
channels[net][relay.num] = []
for channel in relay.channels:
channels[net][relay.num].append(channel)
# debug(f"getAllChannels(): {channels}")
return channels
def getDuplicateChannels(net=None, total=False):
"""
Get a list of duplicate channels.
:return: list of duplicate channels
"""
allChans = getAllChannels(net)
duplicates = {}
for net in allChans.keys():
net_chans = []
inst = {}
# add all the channels from this network to a list
for num in allChans[net].keys():
net_chans.extend(allChans[net][num])
for channel in net_chans:
count_chan = net_chans.count(channel)
# I don't know why but it works
# this is used in userinfo.delChannels
set_min = 1
if total:
set_min = 0
if count_chan > set_min:
inst[channel] = count_chan
if inst:
duplicates[net] = inst
if total:
return duplicates
to_part = {}
for net in allChans:
if net in duplicates:
for num in allChans[net].keys():
for channel in allChans[net][num]:
if channel in duplicates[net].keys():
if duplicates[net][channel] > 1:
if net not in to_part:
to_part[net] = {}
if num not in to_part[net]:
to_part[net][num] = []
to_part[net][num].append(channel)
duplicates[net][channel] -= 1
return to_part
def partChannels(data):
for net in data:
for num in data[net]:
name = f"{net}{num}"
if name in main.IRCPool.keys():
for channel in data[net][num]:
if channel in main.IRCPool[name].channels:
main.IRCPool[name].part(channel)
log(f"Parted {channel} on {net} - {num}")
def getEnabledRelays(net): def getEnabledRelays(net):
""" """
Get a list of enabled relays for a network. Get a list of enabled relays for a network.
@ -241,7 +321,7 @@ def minifyChans(net, listinfo, as_list=False):
if not allRelaysActive(net): if not allRelaysActive(net):
error("All relays for %s are not active, cannot minify list" % net) error("All relays for %s are not active, cannot minify list" % net)
return False return False
for i in getActiveRelays(net): for i in getConnectedRelays(net):
name = net + str(i) name = net + str(i)
for x in main.IRCPool[name].channels: for x in main.IRCPool[name].channels:
if as_list: if as_list:

View File

@ -1,7 +1,8 @@
from twisted.internet.threads import deferToThread from twisted.internet.threads import deferToThread
import main import main
from utils.logging.debug import trace from modules import chankeep
from utils.logging.debug import debug, trace
from utils.logging.log import warn from utils.logging.log import warn
from utils.parsing import parsen from utils.parsing import parsen
@ -289,5 +290,15 @@ def _delChannels(net, channels):
def delChannels(net, channels): # we have left a channel def delChannels(net, channels): # we have left a channel
trace("Purging channel %s for %s" % (", ".join(channels), net)) trace("Purging channel %s for %s" % (", ".join(channels), net))
dupes = chankeep.getDuplicateChannels(net, total=True)
print("dupes: %s" % dupes)
if not dupes:
deferToThread(_delChannels, net, channels)
else:
for channel in channels:
if channel in dupes[net]:
if dupes[net][channel] != 0:
channels.remove(channel)
debug(f"Not removing channel {channel} as {net} has {dupes[net][channel]} other relays covering it")
deferToThread(_delChannels, net, channels) deferToThread(_delChannels, net, channels)
# d.addCallback(testCallback) # d.addCallback(testCallback)