monolith/legacy/modules/chankeep.py

from copy import deepcopy
from math import ceil

from twisted.internet.threads import deferToThread

import main
from modules import helpers
from utils.logging.debug import debug, trace
from utils.logging.log import error, log, warn


def getAllChannels(net=None):
    """
    Get a list of all channels on all relays.
    :return: list of channels
    """
    channels = {}
    if not net:
        nets = main.network.keys()
    else:
        nets = [net]
    for net in nets:
        relays = helpers.get_connected_relays(net)
        for relay in relays:
            if net not in channels:
                channels[net] = {}
            if relay.num not in channels[net]:
                channels[net][relay.num] = []
            for channel in relay.channels:
                channels[net][relay.num].append(channel)
    # debug(f"getAllChannels(): {channels}")
    return channels


def getDuplicateChannels(net=None, total=False):
    """
    Get a list of duplicate channels.
    :return: list of duplicate channels
    """
    allChans = getAllChannels(net)
    duplicates = {}
    for net in allChans.keys():
        net_chans = []
        inst = {}
        # add all the channels from this network to a list
        for num in allChans[net].keys():
            net_chans.extend(allChans[net][num])
        for channel in net_chans:
            count_chan = net_chans.count(channel)
            # I don't know why but it works
            # this is used in userinfo.delChannels
            set_min = 1
            if total:
                set_min = 0
            if count_chan > set_min:
                inst[channel] = count_chan
        if inst:
            duplicates[net] = inst

    if total:
        return duplicates

    to_part = {}
    for net in allChans:
        if net in duplicates:
            for num in allChans[net].keys():
                for channel in allChans[net][num]:
                    if channel in duplicates[net].keys():
                        if duplicates[net][channel] > 1:
                            if net not in to_part:
                                to_part[net] = {}
                            if num not in to_part[net]:
                                to_part[net][num] = []
                            to_part[net][num].append(channel)
                            duplicates[net][channel] -= 1

    return to_part


def partChannels(data):
    for net in data:
        for num in data[net]:
            name = f"{net}{num}"
            if name in main.IRCPool.keys():
                for channel in data[net][num]:
                    if channel in main.IRCPool[name].channels:
                        main.IRCPool[name].part(channel)
                        log(f"Parted {channel} on {net} - {num}")


def getEnabledRelays(net):
    """
    Get a list of enabled relays for a network.
    :param net: network
    :rtype: list of int
    :return: list of enabled relay numbers
    """
    enabledRelays = [x for x in main.network[net].relays.keys() if main.network[net].relays[x]["enabled"]]
    # debug(f"getEnabledRelays() {net}: {enabledRelays}")
    return enabledRelays


def getConnectedRelays(net):
    """
    Get a list of connected relays for a network.
    :param net: network
    :rtype: list of int
    :return: list of relay numbers
    """
    enabledRelays = getEnabledRelays(net)
    connectedRelays = []
    for i in enabledRelays:
        name = net + str(i)
        if name in main.IRCPool.keys():
            if main.IRCPool[name].isconnected:
                connectedRelays.append(i)
    # debug(f"getConnectedRelays() {net}: {connectedRelays}")
    return connectedRelays


def getActiveRelays(net):
    """
    Get a list of active relays for a network.
    :param net: network
    :rtype: list of int
    :return: list of relay numbers
    """
    enabledRelays = getEnabledRelays(net)
    activeRelays = []
    for i in enabledRelays:
        name = net + str(i)
        if name in main.IRCPool.keys():
            # debug(
            #     (
            #         f"getActiveRelays() {net}: {i} auth:{main.IRCPool[name].authenticated} "
            #         f"conn:{main.IRCPool[name].isconnected}"
            #     )
            # )
            if main.IRCPool[name].authenticated and main.IRCPool[name].isconnected:
                activeRelays.append(i)
    debug(f"getActiveRelays() {net}: {activeRelays}")
    return activeRelays


def relayIsActive(net, num):
    """
    Check if a relay is active.
    :param net: network
    :param num: relay number
    :rtype: bool
    :return: True if relay is active, False otherwise
    """
    activeRelays = getActiveRelays(net)
    return num in activeRelays


def allRelaysActive(net):
    """
    Check if all enabled relays are active and authenticated.
    :param net: network
    :rtype: bool
    :return: True if all relays are active and authenticated, False otherwise
    """
    activeRelays = getActiveRelays(net)
    enabledRelays = getEnabledRelays(net)
    relaysActive = len(activeRelays) == len(enabledRelays)
    # debug(f"allRelaysActive() {net}: {relaysActive} ({activeRelays}/{enabledRelays})")
    return relaysActive


def getAverageChanlimit(net):
    """
    Get the average channel limit for a network.
    :param net: network
    :rtype: int
    :return: average channel limit
    """
    total = 0
    for i in getActiveRelays(net):
        name = net + str(i)
        if name in main.IRCPool.keys():
            total += main.IRCPool[name].chanlimit
    avg_chanlimit = total / len(getActiveRelays(net))
    debug(f"getAverageChanlimit() {net}: {avg_chanlimit}")
    return avg_chanlimit


def getSumChanlimit(net):
    """
    Get the sum of all channel limits for a network.
    :param net: network
    :rtype: int
    :return: sum of channel limits
    """
    total = 0
    for i in getActiveRelays(net):
        name = net + str(i)
        if name in main.IRCPool.keys():
            total += main.IRCPool[name].chanlimit
    return total


def getChanFree(net):
    """
    Get a dictionary with the free channel spaces for
    each relay, and a channel limit.
    Example return:
     ({1: 99}, 100)
    :param net: network
    :return: ({relay: channel spaces}, channel limit)
    """
    chanfree = {}
    for i in getActiveRelays(net):
        name = net + str(i)
        if name not in main.IRCPool.keys():
            continue
        if not main.IRCPool[name].isconnected:
            continue
        chanfree[i] = main.IRCPool[name].chanlimit - len(main.IRCPool[name].channels)

    return chanfree


def getTotalChans(net):
    """
    Get the total number of channels on all relays for a network.
    :param net: network
    :rtype: int
    :return: total number of channels
    """
    total = 0
    for i in getActiveRelays(net):
        name = net + str(i)
        if name in main.IRCPool.keys():
            total += len(main.IRCPool[name].channels)
    return total


def emptyChanAllocate(net, flist):
    """
    Allocate channels to relays.
    :param net: network
    :param flist: list of channels to allocate
    :param new: list of newly provisioned relays to account for
    :rtype: dict
    :return: dictionary of {relay: list of channels}"""

    # Get the free channel spaces for each relay
    chanfree = getChanFree(net)
    if not chanfree:
        return

    # Pretend the newly provisioned relays are already on the network
    # for i in new:
    #     chanfree[0][i] = chanfree[1]
    allocated = {}

    newlist = list(flist)
    chan_slots_used = getTotalChans(net)
    max_chans = getSumChanlimit(net) - chan_slots_used
    trunc_list = newlist[:max_chans]
    debug(f"emptyChanAllocate() {net}: newlist:{len(newlist)} trunc_list:{len(trunc_list)}")

    for i in chanfree.keys():
        for x in range(chanfree[i]):
            if not len(trunc_list):
                break
            if i in allocated.keys():
                allocated[i].append(trunc_list.pop())
            else:
                allocated[i] = [trunc_list.pop()]
    return allocated


def populateChans(net, clist):
    """
    Populate channels on relays.
    Stores channels to join in a list in main.TempChan[net][num]
    :param net: network
    :param clist: list of channels to join
    :param new: list of newly provisioned relays to account for"""
    # divided = array_split(clist, relay)
    allocated = emptyChanAllocate(net, clist)
    trace(f"populateChans() allocated:{allocated}")
    if not allocated:
        return
    for i in allocated.keys():
        if net in main.TempChan.keys():
            main.TempChan[net][i] = allocated[i]
        else:
            main.TempChan[net] = {i: allocated[i]}
        trace(f"populateChans() TempChan {net}{i}: {allocated[i]}")


def notifyJoin(net):
    """
    Notify relays to join channels.
    They will pull from main.TempChan and remove channels they join.
    :param net: network
    """
    for i in getActiveRelays(net):
        name = net + str(i)
        if name in main.IRCPool.keys():
            trace(f"notifyJoin() {name}")
            main.IRCPool[name].checkChannels()


def minifyChans(net, listinfo, as_list=False):
    """
    Remove channels from listinfo that are already covered by a relay.
    :param net: network
    :param listinfo: list of channels to check
    :type listinfo: list of [channel, num_users]
    :return: list of channels with joined channels removed
    :rtype: list of [channel, num_users]
    """
    # We want to make this reusable for joining a bunch of channels.
    if as_list:
        channel_list = listinfo

    if not allRelaysActive(net):
        error("All relays for %s are not active, cannot minify list" % net)
        return False
    for i in getConnectedRelays(net):
        name = net + str(i)
        for x in main.IRCPool[name].channels:
            if as_list:
                for y in channel_list:
                    if y == x:
                        channel_list.remove(y)
            else:
                for y in listinfo:
                    if y[0] == x:
                        listinfo.remove(y)
    if not as_list:
        if not listinfo:
            log("We're on all the channels we want to be on, dropping LIST")
            return False
    if as_list:
        return channel_list
    else:
        return listinfo


def keepChannels(net, listinfo, mean, sigrelay, relay):
    """
    Minify channels, determine whether we can cover all the channels
    on the network, or need to use 'significant' mode.
    Truncate the channel list to available channel spaces.
    Allocate these channels to relays.
    Notify relays that they should pull from TempChan to join.
    :param net: network
    :param listinfo: list of [channel, num_users] lists
    :param mean: mean of channel population
    :param sigrelay: number of relays needed to cover significant channels
    :param relay: number of relays needed to cover all channels
    :param chanlimit: maximum number of channels to allocate to a relay
    """
    listinfo = minifyChans(net, listinfo)
    if not listinfo:
        return
    if relay <= main.config["ChanKeep"]["SigSwitch"]:  # we can cover all of the channels
        coverAll = True
    elif relay > main.config["ChanKeep"]["SigSwitch"]:  # we cannot cover all of the channels
        coverAll = False
    #    if not sigrelay <= main.config["ChanKeep"]["MaxRelay"]:
    #        error("Network %s is too big to cover: %i relays required" % (net, sigrelay))
    #        return
    num_instances = len(getActiveRelays(net))
    debug(f"keepChannels() {net} instances:{num_instances}")
    chan_slots_used = getTotalChans(net)
    debug(f"keepChannels() slots_used:{chan_slots_used}")
    # max_chans = (chanlimit * num_instances) - chan_slots_used
    max_chans = getSumChanlimit(net) - chan_slots_used
    if max_chans < 0:
        max_chans = 0
    debug(f"keepChannels() max_chans:{max_chans}")
    if coverAll:
        # needed = relay - len(getActiveRelays(net))
        # if needed:
        #     debug(f"keepChannels() coverAll asking to provision {needed} relays for {net} relay:{relay}")
        #     newNums = modules.provision.provisionMultipleRelays(net, needed)
        # else:
        #     newNums = []
        listinfo_sort = sorted(listinfo, reverse=True, key=lambda x: x[1])
        if len(listinfo_sort) > max_chans:
            max_chans = len(listinfo_sort) - 1

        flist = [i[0] for i in listinfo_sort]

        flist = flist[:max_chans]
        debug(f"keepChannels() {net}: joining {len(flist)}/{len(listinfo_sort)} channels")
        trace(f"keepChannels() {net}: joining:{flist}")
        populateChans(net, flist)
    else:
        # needed = sigrelay - len(getActiveRelays(net))
        # if needed:
        #     debug(f"keepChannels() NOT coverAll asking to provision {needed} relays for {net} sigrelay:{sigrelay}")
        #     newNums = modules.provision.provisionMultipleRelays(net, needed)
        # else:
        #     newNums = []
        listinfo_sort = sorted(listinfo, reverse=True, key=lambda x: x[1])
        trace(f"keepChannels() {net}: listinfo_sort:{listinfo_sort}")
        if len(listinfo_sort) > max_chans:
            max_chans = len(listinfo_sort) - 1
        debug(f"keepChannels() {net}: new max_chans:{max_chans}")

        siglist = [i[0] for i in listinfo if int(i[1]) > mean]
        trace(f"keepChannels() {net}: new siglist:{siglist}")

        siglist = siglist[:max_chans]
        trace(f"keepChannels() {net}: truncated siglist:{siglist}")

        trace(f"keepChannels() {net}: siglist:{siglist} max_chans:{max_chans} len_sig:{len(listinfo_sort)}")
        debug(f"keepChannels() {net}: joining {len(siglist)}/{len(listinfo_sort)} channels")
        trace(f"keepChannels() {net}: joining:{siglist}")
        populateChans(net, siglist)
    notifyJoin(net)


def joinSingle(net, channel):
    """
    Join a channel on a relay.
    Use ECA to determine which relay to join on.
    :param net: network
    :param channel: channel to join
    :return: relay number that joined the channel
    :rtype: int
    """
    if "," in channel:
        channels = channel.split(",")
        channels = minifyChans(net, channels, as_list=True)

    else:
        channels = [channel]
    populateChans(net, channels)
    notifyJoin(net)
    return True


def partSingle(net, channel):
    """
    Iterate over all the relays of net and part channels matching channel.
    :param net: network
    :param channel: channel to part
    :return: list of relays that parted the channel
    :rtype: list of str
    """
    parted = []
    for i in getConnectedRelays(net):
        name = f"{net}{i}"
        if name in main.IRCPool.keys():
            if channel in main.IRCPool[name].channels:
                main.IRCPool[name].part(channel)
                parted.append(str(i))
    return parted


def nukeNetwork(net):
    """
    Remove network records.
    :param net: network"""
    # purgeRecords(net)
    # p = main.g.pipeline()
    main.g.delete("analytics.list." + net)
    # p.delete("list."+net)
    # p.execute()


# def nukeNetwork(net):
#    deferToThread(_nukeNetwork, net)


def _initialList(net, num, listinfo):
    """
    Called when a relay receives a full LIST response.
    Run statistics to determine how many channels are significant.
    This is done by adding all the numbers of users on the channels together,
    then dividing by the number of channels.
    * cumul - cumulative sum of all channel membership
    * siglength - number of significant channels
    * listlength - number of channels in the list
    * sigrelay - number of relays needed to cover siglength
    * relay - number of relays needed to cover all channels
    :param net: network
    :param num: relay number
    :param listinfo: list of [channel, num_users] lists
    :param chanlimit: maximum number of channels the relay can join
    """
    listlength = len(listinfo)
    cumul = 0
    try:
        cumul += sum(int(i[1]) for i in listinfo)
    except TypeError:
        warn("Bad LIST data received from %s - %i" % (net, num))
        return
    mean = round(cumul / listlength, 2)
    siglength = 0
    insiglength = 0
    sigcumul = 0
    insigcumul = 0
    for i in listinfo:
        if int(i[1]) > mean:
            siglength += 1
            sigcumul += int(i[1])
        elif int(i[1]) < mean:
            insiglength += 1
            insigcumul += int(i[1])

    avg_chanlimit = getAverageChanlimit(net)
    sigrelay = ceil(siglength / avg_chanlimit)
    relay = ceil(listlength / avg_chanlimit)

    cur_relays = len(getActiveRelays(net))
    sig_relays_missing = sigrelay - cur_relays
    all_relays_missing = relay - cur_relays

    abase = "analytics.list.%s" % net
    main.g.delete(abase)
    p = main.g.pipeline()

    # See docstring for meanings
    p.hset(abase, "mean", mean)
    p.hset(abase, "total_chans", listlength)
    p.hset(abase, "big_chans", siglength)
    p.hset(abase, "small_chans", insiglength)
    p.hset(abase, "big_chan_perc", round(siglength / listlength * 100, 2))
    p.hset(abase, "small_chan_perc", round(insiglength / listlength * 100, 2))
    p.hset(abase, "total_cumul_mem", cumul)
    p.hset(abase, "big_chan_cumul_mem", sigcumul)
    p.hset(abase, "small_chan_cumul_mem", insigcumul)
    p.hset(abase, "relays_for_all_chans", relay)
    p.hset(abase, "relays_for_big_chans", sigrelay)
    p.hset(abase, "relays_for_small_chans", ceil(insiglength / avg_chanlimit))
    p.hset(abase, "sig_relays_missing", sig_relays_missing)
    p.hset(abase, "all_relays_missing", all_relays_missing)
    debug(
        (
            f"_initialList() net:{net} num:{num} listlength:{listlength} "
            f"mean:{mean} siglength:{siglength} insiglength:{insiglength} "
            f"sigrelay:{sigrelay} relay:{relay} avg_chanlimit:{avg_chanlimit}"
        )
    )

    # Purge existing records before writing
    # purgeRecords(net)
    # for i in listinfo:
    #    p.rpush(netbase+"."+i[0], i[1])
    #    p.rpush(netbase+"."+i[0], i[2])
    #    p.sadd(netbase, i[0])

    p.execute()
    debug("List parsing completed on %s" % net)
    keepChannels(net, listinfo, mean, sigrelay, relay)

    # return (listinfo, mean, sigrelay, relay)


def convert(data):
    """
    Recursively convert a dictionary.
    """
    if isinstance(data, bytes):
        return data.decode("ascii")
    if isinstance(data, dict):
        return dict(map(convert, data.items()))
    if isinstance(data, tuple):
        return map(convert, data)
    if isinstance(data, list):
        return list(map(convert, data))
    return data


def getListInfo(net):
    abase = f"analytics.list.{net}"
    info = main.g.hgetall(abase)
    return convert(info)


def initialList(net, num, listinfo):
    """
    Run _initialList in a thread.
    See above docstring.
    """
    deferToThread(_initialList, net, num, deepcopy(listinfo))