neptune/core/db/processing.py

from datetime import datetime

from core.lib.threshold import annotate_num_chans, annotate_num_users, annotate_online


def annotate_results(results):
    """
    Accept a list of dict objects, search for the number of channels and users.
    Add them to the object.
    Mutate it in place. Does not return anything.
    """
    # Figure out items with net (not discord)
    nets = set()
    for x in results:
        if "net" in x:
            nets.add(x["net"])

    for net in nets:
        # Annotate the online attribute from Threshold
        nicks = list(
            set(
                [
                    x["nick"]
                    for x in results
                    if {"nick", "src", "net"}.issubset(x)
                    and x["src"] == "irc"
                    and x["net"] == net
                ]
            )
        )
        channels = list(
            set(
                [
                    x["channel"]
                    for x in results
                    if {"channel", "src", "net"}.issubset(x)
                    and x["src"] == "irc"
                    and x["net"] == net
                ]
            )
        )
        online_info = None
        num_users = None
        num_chans = None
        if nicks:
            online_info = annotate_online(net, nicks)
        # Annotate the number of users in the channel
        if channels:
            num_users = annotate_num_users(net, channels)
        # Annotate the number channels the user is on
        if nicks:
            num_chans = annotate_num_chans(net, nicks)
        for item in results:
            if "net" in item:
                if item["net"] == net:
                    if "nick" in item:
                        if online_info:
                            if item["nick"] in online_info:
                                item["online"] = online_info[item["nick"]]
                    if "channel" in item:
                        if num_users:
                            if item["channel"] in num_users:
                                item["num_users"] = num_users[item["channel"]]
                    if "nick" in item:
                        if num_chans:
                            if item["nick"] in num_chans:
                                item["num_chans"] = num_chans[item["nick"]]


def parse_results(results, meta=None):
    results_parsed = []
    stringify = ["host", "channel"]
    if "hits" in results.keys():
        if "hits" in results["hits"]:
            for item in results["hits"]["hits"]:
                if "_source" in item.keys():
                    data_index = "_source"
                elif "fields" in item.keys():
                    data_index = "fields"
                else:
                    return False
                element = item[data_index]
                for field in stringify:
                    if field in element:
                        element[field] = str(element[field])
                # Why are fields in lists...
                if data_index == "fields":
                    element = {k: v[0] for k, v in element.items() if len(v)}
                element["id"] = item["_id"]

                # Remove empty values
                for field in list(element.keys()):
                    if element[field] == "":
                        del element[field]

                # Split the timestamp into date and time
                if "ts" not in element:
                    if "time" in element:  # will fix data later
                        ts = element["time"]
                        del element["time"]
                        element["ts"] = ts
                if "ts" in element:
                    if isinstance(element["ts"], str):
                        ts = element["ts"]
                    else:
                        ts = datetime.utcfromtimestamp(element["ts"]).strftime(
                            "%Y-%m-%dT%H:%M:%S"
                        )
                    ts_spl = ts.split("T")
                    date = ts_spl[0]
                    time = ts_spl[1]
                    element["date"] = date
                    if "." in time:
                        time_spl = time.split(".")
                        if len(time_spl) == 2:
                            element["time"] = time.split(".")[0]
                        else:
                            element["time"] = time
                    else:
                        element["time"] = time
                results_parsed.append(element)
    if meta:
        meta = {"aggs": {}}
        if "aggregations" in results:
            for field in ["avg_sentiment"]:  # Add other number fields here
                if field in results["aggregations"]:
                    meta["aggs"][field] = results["aggregations"][field]
        total_hits = results["hits"]["total"]["value"]
        meta["total_hits"] = total_hits
        return (meta, results_parsed)

    return results_parsed


def parse_druid(response):
    results_parsed = []
    for item in response:
        if "events" in item:
            for event in item["events"]:
                results_parsed.append(event)
        else:
            raise Exception(f"events not in item {item}")
    return results_parsed