Implement indexing into Apache Druid #1

Closed
m wants to merge 263 commits from druid into master
4 changed files with 120 additions and 3 deletions
Showing only changes of commit 5cc38da00e - Show all commits

24
commands/dedup.py Normal file
View File

@ -0,0 +1,24 @@
import main
from modules import chankeep
from modules import helpers
from json import dumps
class DedupCommand:
def __init__(self, *args):
self.dedup(*args)
def dedup(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
if authed:
if length == 1:
dupes = chankeep.getDuplicateChannels()
chankeep.partChannels(dupes)
info(dumps(dupes))
return
elif length == 2:
if spl[1] not in main.network.keys():
failure("No such network: %s" % spl[1])
return
dupes = chankeep.getDuplicateChannels(spl[1])
chankeep.partChannels(dupes)
info(dumps(dupes))
return

View File

@ -628,6 +628,8 @@ class IRCBot(IRCClient):
# End of Twisted hackery
def regPing(self, negativepass=None, reset=True):
if not main.config["AutoReg"]:
return
if self.authenticated:
return
if not regproc.needToAuth(self.net):

View File

@ -4,10 +4,90 @@ from math import ceil
from twisted.internet.threads import deferToThread
import main
from modules import helpers
from utils.logging.debug import debug, trace
from utils.logging.log import error, log, warn
def getAllChannels(net=None):
"""
Get a list of all channels on all relays.
:return: list of channels
"""
channels = {}
if not net:
nets = main.network.keys()
else:
nets = [net]
for net in nets:
relays = helpers.get_connected_relays(net)
for relay in relays:
if net not in channels:
channels[net] = {}
if relay.num not in channels[net]:
channels[net][relay.num] = []
for channel in relay.channels:
channels[net][relay.num].append(channel)
# debug(f"getAllChannels(): {channels}")
return channels
def getDuplicateChannels(net=None, total=False):
"""
Get a list of duplicate channels.
:return: list of duplicate channels
"""
allChans = getAllChannels(net)
duplicates = {}
for net in allChans.keys():
net_chans = []
inst = {}
# add all the channels from this network to a list
for num in allChans[net].keys():
net_chans.extend(allChans[net][num])
for channel in net_chans:
count_chan = net_chans.count(channel)
# I don't know why but it works
# this is used in userinfo.delChannels
set_min = 1
if total:
set_min = 0
if count_chan > set_min:
inst[channel] = count_chan
if inst:
duplicates[net] = inst
if total:
return duplicates
to_part = {}
for net in allChans:
if net in duplicates:
for num in allChans[net].keys():
for channel in allChans[net][num]:
if channel in duplicates[net].keys():
if duplicates[net][channel] > 1:
if net not in to_part:
to_part[net] = {}
if num not in to_part[net]:
to_part[net][num] = []
to_part[net][num].append(channel)
duplicates[net][channel] -= 1
return to_part
def partChannels(data):
for net in data:
for num in data[net]:
name = f"{net}{num}"
if name in main.IRCPool.keys():
for channel in data[net][num]:
if channel in main.IRCPool[name].channels:
main.IRCPool[name].part(channel)
log(f"Parted {channel} on {net} - {num}")
def getEnabledRelays(net):
"""
Get a list of enabled relays for a network.
@ -241,7 +321,7 @@ def minifyChans(net, listinfo, as_list=False):
if not allRelaysActive(net):
error("All relays for %s are not active, cannot minify list" % net)
return False
for i in getActiveRelays(net):
for i in getConnectedRelays(net):
name = net + str(i)
for x in main.IRCPool[name].channels:
if as_list:

View File

@ -1,7 +1,8 @@
from twisted.internet.threads import deferToThread
import main
from utils.logging.debug import trace
from modules import chankeep
from utils.logging.debug import debug, trace
from utils.logging.log import warn
from utils.parsing import parsen
@ -289,5 +290,15 @@ def _delChannels(net, channels):
def delChannels(net, channels): # we have left a channel
trace("Purging channel %s for %s" % (", ".join(channels), net))
deferToThread(_delChannels, net, channels)
dupes = chankeep.getDuplicateChannels(net, total=True)
print("dupes: %s" % dupes)
if not dupes:
deferToThread(_delChannels, net, channels)
else:
for channel in channels:
if channel in dupes[net]:
if dupes[net][channel] != 0:
channels.remove(channel)
debug(f"Not removing channel {channel} as {net} has {dupes[net][channel]} other relays covering it")
deferToThread(_delChannels, net, channels)
# d.addCallback(testCallback)