Implement indexing into Apache Druid #1
|
@ -0,0 +1,24 @@
|
||||||
|
import main
|
||||||
|
from modules import chankeep
|
||||||
|
from modules import helpers
|
||||||
|
from json import dumps
|
||||||
|
|
||||||
|
class DedupCommand:
|
||||||
|
def __init__(self, *args):
|
||||||
|
self.dedup(*args)
|
||||||
|
|
||||||
|
def dedup(self, addr, authed, data, obj, spl, success, failure, info, incUsage, length):
|
||||||
|
if authed:
|
||||||
|
if length == 1:
|
||||||
|
dupes = chankeep.getDuplicateChannels()
|
||||||
|
chankeep.partChannels(dupes)
|
||||||
|
info(dumps(dupes))
|
||||||
|
return
|
||||||
|
elif length == 2:
|
||||||
|
if spl[1] not in main.network.keys():
|
||||||
|
failure("No such network: %s" % spl[1])
|
||||||
|
return
|
||||||
|
dupes = chankeep.getDuplicateChannels(spl[1])
|
||||||
|
chankeep.partChannels(dupes)
|
||||||
|
info(dumps(dupes))
|
||||||
|
return
|
|
@ -628,6 +628,8 @@ class IRCBot(IRCClient):
|
||||||
# End of Twisted hackery
|
# End of Twisted hackery
|
||||||
|
|
||||||
def regPing(self, negativepass=None, reset=True):
|
def regPing(self, negativepass=None, reset=True):
|
||||||
|
if not main.config["AutoReg"]:
|
||||||
|
return
|
||||||
if self.authenticated:
|
if self.authenticated:
|
||||||
return
|
return
|
||||||
if not regproc.needToAuth(self.net):
|
if not regproc.needToAuth(self.net):
|
||||||
|
|
|
@ -4,10 +4,90 @@ from math import ceil
|
||||||
from twisted.internet.threads import deferToThread
|
from twisted.internet.threads import deferToThread
|
||||||
|
|
||||||
import main
|
import main
|
||||||
|
from modules import helpers
|
||||||
from utils.logging.debug import debug, trace
|
from utils.logging.debug import debug, trace
|
||||||
from utils.logging.log import error, log, warn
|
from utils.logging.log import error, log, warn
|
||||||
|
|
||||||
|
|
||||||
|
def getAllChannels(net=None):
|
||||||
|
"""
|
||||||
|
Get a list of all channels on all relays.
|
||||||
|
:return: list of channels
|
||||||
|
"""
|
||||||
|
channels = {}
|
||||||
|
if not net:
|
||||||
|
nets = main.network.keys()
|
||||||
|
else:
|
||||||
|
nets = [net]
|
||||||
|
for net in nets:
|
||||||
|
relays = helpers.get_connected_relays(net)
|
||||||
|
for relay in relays:
|
||||||
|
if net not in channels:
|
||||||
|
channels[net] = {}
|
||||||
|
if relay.num not in channels[net]:
|
||||||
|
channels[net][relay.num] = []
|
||||||
|
for channel in relay.channels:
|
||||||
|
channels[net][relay.num].append(channel)
|
||||||
|
# debug(f"getAllChannels(): {channels}")
|
||||||
|
return channels
|
||||||
|
|
||||||
|
|
||||||
|
def getDuplicateChannels(net=None, total=False):
|
||||||
|
"""
|
||||||
|
Get a list of duplicate channels.
|
||||||
|
:return: list of duplicate channels
|
||||||
|
"""
|
||||||
|
allChans = getAllChannels(net)
|
||||||
|
duplicates = {}
|
||||||
|
for net in allChans.keys():
|
||||||
|
net_chans = []
|
||||||
|
inst = {}
|
||||||
|
# add all the channels from this network to a list
|
||||||
|
for num in allChans[net].keys():
|
||||||
|
net_chans.extend(allChans[net][num])
|
||||||
|
for channel in net_chans:
|
||||||
|
count_chan = net_chans.count(channel)
|
||||||
|
# I don't know why but it works
|
||||||
|
# this is used in userinfo.delChannels
|
||||||
|
set_min = 1
|
||||||
|
if total:
|
||||||
|
set_min = 0
|
||||||
|
if count_chan > set_min:
|
||||||
|
inst[channel] = count_chan
|
||||||
|
if inst:
|
||||||
|
duplicates[net] = inst
|
||||||
|
|
||||||
|
if total:
|
||||||
|
return duplicates
|
||||||
|
|
||||||
|
to_part = {}
|
||||||
|
for net in allChans:
|
||||||
|
if net in duplicates:
|
||||||
|
for num in allChans[net].keys():
|
||||||
|
for channel in allChans[net][num]:
|
||||||
|
if channel in duplicates[net].keys():
|
||||||
|
if duplicates[net][channel] > 1:
|
||||||
|
if net not in to_part:
|
||||||
|
to_part[net] = {}
|
||||||
|
if num not in to_part[net]:
|
||||||
|
to_part[net][num] = []
|
||||||
|
to_part[net][num].append(channel)
|
||||||
|
duplicates[net][channel] -= 1
|
||||||
|
|
||||||
|
return to_part
|
||||||
|
|
||||||
|
|
||||||
|
def partChannels(data):
|
||||||
|
for net in data:
|
||||||
|
for num in data[net]:
|
||||||
|
name = f"{net}{num}"
|
||||||
|
if name in main.IRCPool.keys():
|
||||||
|
for channel in data[net][num]:
|
||||||
|
if channel in main.IRCPool[name].channels:
|
||||||
|
main.IRCPool[name].part(channel)
|
||||||
|
log(f"Parted {channel} on {net} - {num}")
|
||||||
|
|
||||||
|
|
||||||
def getEnabledRelays(net):
|
def getEnabledRelays(net):
|
||||||
"""
|
"""
|
||||||
Get a list of enabled relays for a network.
|
Get a list of enabled relays for a network.
|
||||||
|
@ -241,7 +321,7 @@ def minifyChans(net, listinfo, as_list=False):
|
||||||
if not allRelaysActive(net):
|
if not allRelaysActive(net):
|
||||||
error("All relays for %s are not active, cannot minify list" % net)
|
error("All relays for %s are not active, cannot minify list" % net)
|
||||||
return False
|
return False
|
||||||
for i in getActiveRelays(net):
|
for i in getConnectedRelays(net):
|
||||||
name = net + str(i)
|
name = net + str(i)
|
||||||
for x in main.IRCPool[name].channels:
|
for x in main.IRCPool[name].channels:
|
||||||
if as_list:
|
if as_list:
|
||||||
|
|
|
@ -1,7 +1,8 @@
|
||||||
from twisted.internet.threads import deferToThread
|
from twisted.internet.threads import deferToThread
|
||||||
|
|
||||||
import main
|
import main
|
||||||
from utils.logging.debug import trace
|
from modules import chankeep
|
||||||
|
from utils.logging.debug import debug, trace
|
||||||
from utils.logging.log import warn
|
from utils.logging.log import warn
|
||||||
from utils.parsing import parsen
|
from utils.parsing import parsen
|
||||||
|
|
||||||
|
@ -289,5 +290,15 @@ def _delChannels(net, channels):
|
||||||
|
|
||||||
def delChannels(net, channels): # we have left a channel
|
def delChannels(net, channels): # we have left a channel
|
||||||
trace("Purging channel %s for %s" % (", ".join(channels), net))
|
trace("Purging channel %s for %s" % (", ".join(channels), net))
|
||||||
deferToThread(_delChannels, net, channels)
|
dupes = chankeep.getDuplicateChannels(net, total=True)
|
||||||
|
print("dupes: %s" % dupes)
|
||||||
|
if not dupes:
|
||||||
|
deferToThread(_delChannels, net, channels)
|
||||||
|
else:
|
||||||
|
for channel in channels:
|
||||||
|
if channel in dupes[net]:
|
||||||
|
if dupes[net][channel] != 0:
|
||||||
|
channels.remove(channel)
|
||||||
|
debug(f"Not removing channel {channel} as {net} has {dupes[net][channel]} other relays covering it")
|
||||||
|
deferToThread(_delChannels, net, channels)
|
||||||
# d.addCallback(testCallback)
|
# d.addCallback(testCallback)
|
||||||
|
|
Loading…
Reference in New Issue