Implement indexing into Apache Druid #1
|
@ -0,0 +1,24 @@
|
|||
import main
|
||||
from modules import chankeep
|
||||
from modules import helpers
|
||||
from json import dumps
|
||||
|
||||
class DedupCommand:
|
||||
def __init__(self, *args):
|
||||
self.dedup(*args)
|
||||
|
||||
def dedup(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
|
||||
if authed:
|
||||
if length == 1:
|
||||
dupes = chankeep.getDuplicateChannels()
|
||||
chankeep.partChannels(dupes)
|
||||
info(dumps(dupes))
|
||||
return
|
||||
elif length == 2:
|
||||
if spl[1] not in main.network.keys():
|
||||
failure("No such network: %s" % spl[1])
|
||||
return
|
||||
dupes = chankeep.getDuplicateChannels(spl[1])
|
||||
chankeep.partChannels(dupes)
|
||||
info(dumps(dupes))
|
||||
return
|
|
@ -628,6 +628,8 @@ class IRCBot(IRCClient):
|
|||
# End of Twisted hackery
|
||||
|
||||
def regPing(self, negativepass=None, reset=True):
|
||||
if not main.config["AutoReg"]:
|
||||
return
|
||||
if self.authenticated:
|
||||
return
|
||||
if not regproc.needToAuth(self.net):
|
||||
|
|
|
@ -4,10 +4,90 @@ from math import ceil
|
|||
from twisted.internet.threads import deferToThread
|
||||
|
||||
import main
|
||||
from modules import helpers
|
||||
from utils.logging.debug import debug, trace
|
||||
from utils.logging.log import error, log, warn
|
||||
|
||||
|
||||
def getAllChannels(net=None):
|
||||
"""
|
||||
Get a list of all channels on all relays.
|
||||
:return: list of channels
|
||||
"""
|
||||
channels = {}
|
||||
if not net:
|
||||
nets = main.network.keys()
|
||||
else:
|
||||
nets = [net]
|
||||
for net in nets:
|
||||
relays = helpers.get_connected_relays(net)
|
||||
for relay in relays:
|
||||
if net not in channels:
|
||||
channels[net] = {}
|
||||
if relay.num not in channels[net]:
|
||||
channels[net][relay.num] = []
|
||||
for channel in relay.channels:
|
||||
channels[net][relay.num].append(channel)
|
||||
# debug(f"getAllChannels(): {channels}")
|
||||
return channels
|
||||
|
||||
|
||||
def getDuplicateChannels(net=None, total=False):
|
||||
"""
|
||||
Get a list of duplicate channels.
|
||||
:return: list of duplicate channels
|
||||
"""
|
||||
allChans = getAllChannels(net)
|
||||
duplicates = {}
|
||||
for net in allChans.keys():
|
||||
net_chans = []
|
||||
inst = {}
|
||||
# add all the channels from this network to a list
|
||||
for num in allChans[net].keys():
|
||||
net_chans.extend(allChans[net][num])
|
||||
for channel in net_chans:
|
||||
count_chan = net_chans.count(channel)
|
||||
# I don't know why but it works
|
||||
# this is used in userinfo.delChannels
|
||||
set_min = 1
|
||||
if total:
|
||||
set_min = 0
|
||||
if count_chan > set_min:
|
||||
inst[channel] = count_chan
|
||||
if inst:
|
||||
duplicates[net] = inst
|
||||
|
||||
if total:
|
||||
return duplicates
|
||||
|
||||
to_part = {}
|
||||
for net in allChans:
|
||||
if net in duplicates:
|
||||
for num in allChans[net].keys():
|
||||
for channel in allChans[net][num]:
|
||||
if channel in duplicates[net].keys():
|
||||
if duplicates[net][channel] > 1:
|
||||
if net not in to_part:
|
||||
to_part[net] = {}
|
||||
if num not in to_part[net]:
|
||||
to_part[net][num] = []
|
||||
to_part[net][num].append(channel)
|
||||
duplicates[net][channel] -= 1
|
||||
|
||||
return to_part
|
||||
|
||||
|
||||
def partChannels(data):
|
||||
for net in data:
|
||||
for num in data[net]:
|
||||
name = f"{net}{num}"
|
||||
if name in main.IRCPool.keys():
|
||||
for channel in data[net][num]:
|
||||
if channel in main.IRCPool[name].channels:
|
||||
main.IRCPool[name].part(channel)
|
||||
log(f"Parted {channel} on {net} - {num}")
|
||||
|
||||
|
||||
def getEnabledRelays(net):
|
||||
"""
|
||||
Get a list of enabled relays for a network.
|
||||
|
@ -241,7 +321,7 @@ def minifyChans(net, listinfo, as_list=False):
|
|||
if not allRelaysActive(net):
|
||||
error("All relays for %s are not active, cannot minify list" % net)
|
||||
return False
|
||||
for i in getActiveRelays(net):
|
||||
for i in getConnectedRelays(net):
|
||||
name = net + str(i)
|
||||
for x in main.IRCPool[name].channels:
|
||||
if as_list:
|
||||
|
|
|
@ -1,7 +1,8 @@
|
|||
from twisted.internet.threads import deferToThread
|
||||
|
||||
import main
|
||||
from utils.logging.debug import trace
|
||||
from modules import chankeep
|
||||
from utils.logging.debug import debug, trace
|
||||
from utils.logging.log import warn
|
||||
from utils.parsing import parsen
|
||||
|
||||
|
@ -289,5 +290,15 @@ def _delChannels(net, channels):
|
|||
|
||||
def delChannels(net, channels): # we have left a channel
|
||||
trace("Purging channel %s for %s" % (", ".join(channels), net))
|
||||
dupes = chankeep.getDuplicateChannels(net, total=True)
|
||||
print("dupes: %s" % dupes)
|
||||
if not dupes:
|
||||
deferToThread(_delChannels, net, channels)
|
||||
else:
|
||||
for channel in channels:
|
||||
if channel in dupes[net]:
|
||||
if dupes[net][channel] != 0:
|
||||
channels.remove(channel)
|
||||
debug(f"Not removing channel {channel} as {net} has {dupes[net][channel]} other relays covering it")
|
||||
deferToThread(_delChannels, net, channels)
|
||||
# d.addCallback(testCallback)
|
||||
|
|
Loading…
Reference in New Issue