pluto/handler/transactions.py

# Twisted/Klein imports
from twisted.internet.task import LoopingCall
from twisted.internet.threads import deferToThread

# Other library imports
from json import dumps
from random import choices
from string import ascii_uppercase
from elasticsearch import Elasticsearch
from datetime import datetime
import urllib3
import logging

# Project imports
from settings import settings
from db import r
import util

# TODO: secure ES traffic properly
urllib3.disable_warnings()

tracer = logging.getLogger("elasticsearch")
tracer.setLevel(logging.CRITICAL)
tracer = logging.getLogger("elastic_transport.transport")
tracer.setLevel(logging.CRITICAL)


class Transactions(util.Base):
    """
    Handler class for incoming Revolut transactions.
    """

    def __init__(self):
        """
        Initialise the Transaction object.
        Set the logger.
        """
        super().__init__()
        if settings.ES.Enabled == "1":
            self.es = Elasticsearch(
                f"https://{settings.ES.Host}:9200",
                verify_certs=False,
                basic_auth=(settings.ES.Username, settings.ES.Pass),
                # ssl_assert_fingerprint=("6b264fd2fd107d45652d8add1750a8a78f424542e13b056d0548173006260710"),
                ca_certs="certs/ca.crt",
            )

    def run_checks_in_thread(self):
        """
        Run all the balance checks that output into ES in another thread.
        """
        deferToThread(self.get_total)
        deferToThread(self.get_remaining)
        deferToThread(self.money.get_profit)
        deferToThread(self.money.get_profit, True)
        deferToThread(self.get_open_trades_usd)
        deferToThread(self.get_total_remaining)
        deferToThread(self.get_total_with_trades)

    def setup_loops(self):
        """
        Set up the LoopingCalls to get the balance so we have data in ES.
        """
        if settings.ES.Enabled == "1":
            self.lc_es_checks = LoopingCall(self.run_checks_in_thread)
            delay = int(settings.ES.RefreshSec)
            self.lc_es_checks.start(delay)
            self.agora.es = self.es

    def valid_transaction(self, data):
        """
        Determine if a given transaction object is valid.
        :param data: a transaction cast
        :type data: dict
        :return: whether the transaction is valid
        :rtype: bool
        """
        txid = data["transaction_id"]
        if "amount" not in data:
            return False
        if "currency" not in data:
            return False
        amount = data["amount"]
        if amount <= 0:
            self.log.info(f"Ignoring transaction with negative/zero amount: {txid}")
            return False
        return True

    def extract_reference(self, data):
        """
        Extract a reference from the transaction cast.
        :param data: a transaction cast
        :type data: dict
        :return: the extracted reference or not_set
        :rtype: str
        """
        if "reference" in data:
            return data["reference"]
        elif "meta" in data:
            if "provider_reference" in data["meta"]:
                return data["meta"]["provider_reference"]
        return "not_set"

    def extract_sender(self, data):
        """
        Extract a sender name from the transaction cast.
        :param data: a transaction cast
        :type data: dict
        :return: the sender name or not_set
        :rtype: str
        """
        if "debtorName" in data:
            return data["debtorName"]
        elif "meta" in data:
            if "debtor_account_name" in data["meta"]:
                return data["meta"]["debtor_account_name"]
        return "not_set"

    def reference_partial_check(self, reference, txid, currency, amount):
        """
        Perform a partial check by intersecting all parts of the split of the
        reference against the existing references, and returning a set of the matches.
        :param reference: the reference to check
        :type reference: str
        :return: matching trade ID string
        :rtype: str
        """
        # Partial reference implementation
        # Account for silly people not removing the default string
        # Split the reference into parts
        ref_split = reference.split(" ")
        # Get all existing references
        existing_refs = self.get_refs()
        # Get all parts of the given reference split that match the existing references
        stored_trade_reference = set(existing_refs).intersection(set(ref_split))
        if len(stored_trade_reference) > 1:
            self.log.error(f"Multiple references valid for TXID {txid}: {reference}")
            self.irc.sendmsg(f"Multiple references valid for TXID {txid}: {reference}")
            self.ux.notify.notify_tx_lookup_failed(currency, amount, reference, "MULTIPLE_REFS_MATCH")
            return False
        if len(stored_trade_reference) == 0:
            return None
        return stored_trade_reference.pop()

    def can_alt_lookup(self, amount, currency, reference):
        amount_usd = self.money.to_usd(amount, currency)
        # Amount is reliable here as it is checked by find_trade, so no need for stored_trade["amount"]
        if float(amount_usd) > float(settings.Agora.AcceptableAltLookupUSD):
            self.log.info("Not checking against amount and currency as amount exceeds MAX")
            self.irc.sendmsg(f"Not checking against amount and currency as amount exceeds MAX")
            # Close here if the amount exceeds the allowable limit for no reference
            self.ux.notify.notify_tx_lookup_failed(currency, amount, reference, "EXCEEDS_MAX")
            return False
        return True

    def amount_currency_lookup(self, amount, currency, txid, reference):
        self.log.info(f"No reference in DB refs for {reference}")
        self.irc.sendmsg(f"No reference in DB refs for {reference}")
        # Try checking just amount and currency, as some people (usually people buying small amounts)
        #  are unable to put in a reference properly.

        self.log.info(f"Checking against amount and currency for TXID {txid}")
        self.irc.sendmsg(f"Checking against amount and currency for TXID {txid}")
        if not self.can_alt_lookup(amount, currency, reference):
            return False
        stored_trade = self.find_trade(txid, currency, amount)
        if not stored_trade:
            self.log.info(f"Failed to get reference by amount and currency: {txid} {currency} {amount}")
            self.irc.sendmsg(f"Failed to get reference by amount and currency: {txid} {currency} {amount}")
            self.ux.notify.notify_tx_lookup_failed(currency, amount, reference, "ALT_LOOKUP_FAILED")
            return None
        stored_trade["amount"] = float(stored_trade["amount"])  # convert to float
        return stored_trade

    def normal_lookup(self, stored_trade_reference, reference, currency, amount):
        stored_trade = self.get_ref(stored_trade_reference)
        if not stored_trade:
            self.log.info(f"No reference in DB for {reference}")
            self.irc.sendmsg(f"No reference in DB for {reference}")
            self.ux.notify.notify_tx_lookup_failed(currency, amount, reference, "NOREF", stored_trade_reference)
            return False
        stored_trade["amount"] = float(stored_trade["amount"])  # convert to float
        return stored_trade

    def currency_check(self, currency, amount, reference, stored_trade):
        if not stored_trade["currency"] == currency:
            self.log.info(f"Currency mismatch, Agora: {stored_trade['currency']} / Sink: {currency}")
            self.irc.sendmsg(f"Currency mismatch, Agora: {stored_trade['currency']} / Sink: {currency}")
            self.ux.notify.notify_tx_lookup_failed(
                currency,
                amount,
                reference,
                "CURRENCY_MISMATCH",
                stored_trade["id"],
            )
            return False
        return True

    def alt_amount_check(self, platform, amount, currency, reference, stored_trade):
        # If the amount does not match exactly, get the min and max values for our given acceptable margins for trades
        min_amount, max_amount = self.money.get_acceptable_margins(platform, currency, stored_trade["amount"])
        self.log.info(f"Amount does not match exactly, trying with margins: min: {min_amount} / max: {max_amount}")
        self.irc.sendmsg(f"Amount does not match exactly, trying with margins: min: {min_amount} / max: {max_amount}")
        if not min_amount < amount < max_amount:
            self.log.info(
                "Amount mismatch - not in margins: {stored_trade['amount']} (min: {min_amount} / max: {max_amount}"
            )
            self.irc.sendmsg(
                f"Amount mismatch - not in margins: {stored_trade['amount']} (min: {min_amount} / max: {max_amount}"
            )
            self.ux.notify.notify_tx_lookup_failed(
                currency,
                amount,
                reference,
                "AMOUNT_MARGIN_MISMATCH",
                stored_trade["id"],
            )
            return False
        return True

    def add_bank_sender(self, platform, platform_buyer, bank_sender):
        """
        Add the bank senders into Redis.
        :param platform: name of the platform - freeform
        :param platform_buyer: the username of the buyer on the platform
        :param bank_sender: the sender name from the bank
        """
        key = f"namemap.{platform}.{platform_buyer}"
        r.sadd(key, bank_sender)

    def get_previous_senders(self, platform, platform_buyer):
        """
        Get all the previous bank sender names for the given buyer on the platform.
        :param platform: name of the platform - freeform
        :param platform_buyer: the username of the buyer on the platform
        :return: set of previous buyers
        :rtype: set
        """
        key = f"namemap.{platform}.{platform_buyer}"
        senders = r.smembers(key)
        if not senders:
            return None
        senders = util.convert(senders)
        return senders

    # def get_name_derivations(self, )

    def check_valid_sender(self, reference, platform, bank_sender, platform_buyer):
        """
        Check that either:
        * The platform buyer has never had a recognised transaction before
        * The bank sender name matches a previous transaction from the platform buyer
        :param reference: the trade reference
        :param platform: name of the platform - freeform
        :param bank_sender: the sender of the bank transaction
        :param platform_buyer: the username of the buyer on the platform
        :return: whether the sender is valid
        :rtype: bool
        """
        senders = self.get_previous_senders(platform, platform_buyer)
        if senders is None:  # no senders yet, assume it's valid
            print("Senders is none, assuming its a valid TX!")
            return True
        if platform_buyer in senders:
            print("Platform buyer is in senders!")
            return True
        print("Platform buyer is not in senders")
        self.ux.notify.notify_sender_name_mismatch(reference, platform_buyer, bank_sender)
        return False

    def check_tx_sender(self, tx, reference):
        """
        Check whether the sender of a given transaction is authorised based on the previous
        transactions of the username that originated the trade reference.
        :param tx: the transaction ID
        :param reference: the trade reference
        """
        stored_trade = self.get_ref(reference)
        if not stored_trade:
            return None
        stored_tx = self.get_tx(tx)
        if not stored_tx:
            return None
        bank_sender = stored_tx["sender"]
        platform_buyer = stored_trade["buyer"]
        platform = stored_trade["subclass"]
        is_allowed = self.check_valid_sender(reference, platform, bank_sender, platform_buyer)
        if is_allowed is True:
            return True
        return False

    def update_trade_tx(self, reference, txid):
        """
        Update a trade to point to a given transaction ID.
        Return False if the trade already has a mapped transaction.
        """
        existing_tx = r.hget(f"trade.{reference}", "tx")
        print("existing_tx", existing_tx)
        if existing_tx is None:
            return None
        elif existing_tx == b"":
            r.hset(f"trade.{reference}", "tx", txid)
            return True
        else:  # Already a mapped transaction
            return False

    def transaction(self, data):
        """
        Store details of transaction and post notifications to IRC.
        Matches it up with data stored in Redis to attempt to reconcile with an Agora trade.
        :param data: details of transaction
        :type data: dict
        """
        valid = self.valid_transaction(data)
        if not valid:
            return False
        print(f"Raw transaction data: {data}")
        ts = data["timestamp"]
        txid = data["transaction_id"]
        amount = float(data["amount"])
        currency = data["currency"]

        reference = self.extract_reference(data)
        sender = self.extract_sender(data)

        subclass = data["subclass"]
        to_store = {
            "subclass": subclass,
            "ts": ts,
            "txid": txid,
            "reference": reference,
            "amount": amount,
            "currency": currency,
            "sender": sender,
        }
        r.hmset(f"tx.{txid}", to_store)

        self.log.info(f"Transaction processed: {dumps(to_store, indent=2)}")
        self.irc.sendmsg(f"AUTO Incoming transaction on {subclass}: {txid} {amount}{currency} ({reference})")

        stored_trade_reference = self.reference_partial_check(reference, txid, currency, amount)
        if stored_trade_reference is False:  # can be None though
            return

        stored_trade = False
        looked_up_without_reference = False

        # Normal implementation for when we have a reference
        if stored_trade_reference:
            stored_trade = self.normal_lookup(stored_trade_reference, reference, currency, amount)
            # if not stored_trade:
            #    return

        # Amount/currency lookup implementation for when we have no reference
        else:
            if not stored_trade:  # check we don't overwrite the lookup above
                stored_trade = self.amount_currency_lookup(amount, currency, txid, reference)
                if stored_trade is False:
                    return
                if stored_trade:
                    # Note that we have looked it up without reference so we don't use +- below
                    # This might be redundant given the amount checks in find_trade, but better safe than sorry!
                    looked_up_without_reference = True
                else:
                    return
            else:
                # Stored trade reference is none, the checks below will do nothing at all
                return

        # Make sure it was sent in the expected currency
        if not self.currency_check(currency, amount, reference, stored_trade):
            return

        # Make sure the expected amount was sent
        if not stored_trade["amount"] == amount:
            if looked_up_without_reference:
                return
            platform = stored_trade["subclass"]
            if not self.alt_amount_check(platform, amount, currency, reference, stored_trade):
                return
        platform = stored_trade["subclass"]
        platform_buyer = stored_trade["buyer"]

        # Check sender - we don't do anything with this yet
        sender_valid = self.check_valid_sender(reference, platform, sender, platform_buyer)
        print("Sender valid for trade: ", sender_valid)
        # trade_released = self.release_map_trade(reference, txid)
        # if trade_released:
        #    self.ux.notify.notify_complete_trade(amount, currency)
        # else:
        #    self.log.error(f"Cannot release trade {reference}.")
        #    return

        self.release_funds(stored_trade["id"], stored_trade["reference"])

    def release_funds(self, trade_id, reference):
        stored_trade = self.get_ref(reference)
        platform = stored_trade["subclass"]
        logmessage = f"All checks passed, releasing funds for {trade_id} {reference}"
        self.log.info(logmessage)
        self.irc.sendmsg(logmessage)
        if platform == "agora":
            release = self.agora.release_funds
            post_message = self.agora.agora.contact_message_post
        elif platform == "lbtc":
            release = self.lbtc.release_funds
            post_message = self.lbtc.lbtc.contact_message_post

        rtrn = release(trade_id)
        if rtrn["message"] == "OK":
            post_message(trade_id, "Thanks! Releasing now :)")
        else:
            logmessage = f"Release funds unsuccessful: {rtrn['message']}"
            self.log.error(logmessage)
            self.irc.sendmsg(logmessage)
            self.ux.notify.notify_release_unsuccessful(trade_id)
            return

        # Parse the escrow release response
        message = rtrn["message"]
        # message_long = rtrn["response"]["data"]["message"]
        self.irc.sendmsg(f"{dumps(message)}")

    def release_map_trade(self, reference, tx):
        """
        Map a trade to a transaction and release if no other TX is
        mapped to the same trade.
        """
        print("Release map trade called", reference, tx)
        stored_trade = self.get_ref(reference)
        print("Stored trade", stored_trade)
        if not stored_trade:
            self.log.error(f"Could not get stored trade for {reference}.")
            return None
        tx_obj = self.get_tx(tx)
        print("tx_obj", tx_obj)
        if not tx_obj:
            self.log.error(f"Could not get TX for {tx}.")
            return None
        platform = stored_trade["subclass"]
        platform_buyer = stored_trade["buyer"]
        bank_sender = tx_obj["sender"]
        trade_id = stored_trade["id"]
        is_updated = self.update_trade_tx(reference, tx)
        print("is_updated", is_updated)
        if is_updated is None:
            return None
        elif is_updated is True:
            # We mapped the trade successfully
            self.release_funds(trade_id, reference)
            print("Adding mapped bank sender", platform_buyer, bank_sender)
            self.add_bank_sender(platform, platform_buyer, bank_sender)
            return True
        elif is_updated is False:
            # Already mapped
            self.log.error(f"Trade {reference} already has a TX mapped, cannot map {tx}.")
        return False

    def find_trades_by_uid(self, uid):
        """
        Find a list of trade IDs and references by a customer UID.
        :return: tuple of (platform, trade_id, reference, currency)
        """
        platform, username = self.get_uid(uid)
        refs = self.get_refs()
        matching_trades = []
        for reference in refs:
            ref_data = self.get_ref(reference)
            tx_platform = ref_data["subclass"]
            tx_username = ref_data["buyer"]
            trade_id = ref_data["id"]
            currency = ref_data["currency"]
            if tx_platform == platform and tx_username == username:
                to_append = (platform, trade_id, reference, currency)
                matching_trades.append(to_append)
        return matching_trades

    def user_verification_successful(self, uid):
        """
        A user has successfully completed verification.
        """
        self.log.info(f"User has completed verification: {uid}")
        trade_list = self.find_trades_by_uid(uid)
        for platform, trade_id, reference, currency in trade_list:
            self.send_bank_details(platform, currency, trade_id)
            self.send_reference(platform, trade_id, reference)

    def create_uid(self, platform, username):
        return f"{platform}|{username}"

    def get_uid(self, external_user_id):
        """
        Get the platform and username from the external user ID.
        """
        spl = external_user_id.split("|")
        if not len(spl) == 2:
            self.log.error(f"Split invalid, cannot get customer: {spl}")
            return False
        platform, username = spl
        return (platform, username)

    def get_send_settings(self, platform):
        if platform == "agora":
            send_setting = settings.Agora.Send
            post_message = self.agora.agora.contact_message_post
        elif platform == "lbtc":
            send_setting = settings.LocalBitcoins.Send
            post_message = self.lbtc.lbtc.contact_message_post

        return (send_setting, post_message)

    def send_reference(self, platform, trade_id, reference):
        """
        Send the reference to a customer.
        """
        send_setting, post_message = self.get_send_settings(platform)
        if send_setting == "1":
            post_message(
                trade_id,
                f"When sending the payment please use reference code: {reference}",
            )

    def send_verification_url(self, platform, uid, trade_id):
        send_setting, post_message = self.get_send_settings(platform)
        if send_setting == "1":
            auth_url = self.ux.verify.create_applicant_and_get_link(uid)
            post_message(
                trade_id,
                f"Hi! To continue the trade, please complete the verification form: {auth_url}",
            )

    def send_bank_details(self, platform, currency, trade_id):
        """
        Send the bank details to a trade.
        """
        send_setting, post_message = self.get_send_settings(platform)
        self.log.info(f"Sending bank details/reference for {platform}/{trade_id}")
        if send_setting == "1":
            account_info = self.markets.get_matching_account_details(platform, currency)
            formatted_account_info = self.markets.format_payment_details(currency, account_info, real=True)
            post_message(
                trade_id,
                f"Payment details: \n{formatted_account_info}",
            )

    def new_trade(
        self,
        subclass,
        asset,
        trade_id,
        buyer,
        currency,
        amount,
        amount_crypto,
        provider,
    ):
        """
        Called when we have a new trade in Agora.
        Store details in Redis, generate a reference and optionally let the customer know the reference.
        """
        reference = "".join(choices(ascii_uppercase, k=5))
        reference = f"PGN-{reference}"
        existing_ref = r.get(f"trade.{trade_id}.reference")
        if not existing_ref:
            to_store = {
                "id": trade_id,
                "tx": "",
                "asset": asset,
                "buyer": buyer,
                "currency": currency,
                "amount": amount,
                "amount_crypto": amount_crypto,
                "reference": reference,
                "provider": provider,
                "subclass": subclass,
            }
            self.log.info(f"Storing trade information: {str(to_store)}")
            r.hmset(f"trade.{reference}", to_store)
            r.set(f"trade.{trade_id}.reference", reference)
            self.irc.sendmsg(f"Generated reference for {trade_id}: {reference}")
            self.ux.notify.notify_new_trade(amount, currency)
            uid = self.create_uid(subclass, buyer)
            print("UID of new trade", uid)
            verified = self.ux.verify.get_external_user_id_status(uid)
            if verified != "GREEN":
                self.log.info(f"UID {uid} is not verified, sending link.")
                self.send_verification_url(subclass, uid, trade_id)
            else:  # User is verified
                self.log.info(f"UID {uid} is verified.")
                self.send_bank_details(subclass, currency, trade_id)
                self.send_reference(subclass, trade_id, reference)
        if existing_ref:
            return util.convert(existing_ref)
        else:
            return reference

    def find_trade(self, txid, currency, amount):
        """
        Get a trade reference that matches the given currency and amount.
        Only works if there is one result.
        :param txid: Sink transaction ID
        :param currency: currency
        :param amount: amount
        :type txid: string
        :type currency: string
        :type amount: int
        :return: matching trade object or False
        :rtype: dict or bool
        """
        refs = self.get_refs()
        matching_refs = []
        # TODO: use get_ref_map in this function instead of calling get_ref multiple times
        for ref in refs:
            stored_trade = self.get_ref(ref)
            if stored_trade["currency"] == currency and float(stored_trade["amount"]) == float(amount):
                matching_refs.append(stored_trade)
        if len(matching_refs) != 1:
            self.log.error(f"Find trade returned multiple results for TXID {txid}: {matching_refs}")
            return False
        return matching_refs[0]

    def get_refs(self):
        """
        Get all reference IDs for trades.
        :return: list of trade IDs
        :rtype: list
        """
        references = []
        ref_keys = r.keys("trade.*.reference")
        for key in ref_keys:
            references.append(r.get(key))
        return util.convert(references)

    def get_ref_map(self):
        """
        Get all reference IDs for trades.
        :return: dict of references keyed by TXID
        :rtype: dict
        """
        references = {}
        ref_keys = r.keys("trade.*.reference")
        for key in ref_keys:
            tx = util.convert(key).split(".")[1]
            references[tx] = r.get(key)
        return util.convert(references)

    def get_ref(self, reference):
        """
        Get the trade information for a reference.
        :param reference: trade reference
        :type reference: string
        :return: dict of trade information
        :rtype: dict
        """
        ref_data = r.hgetall(f"trade.{reference}")
        ref_data = util.convert(ref_data)
        if "subclass" not in ref_data:
            ref_data["subclass"] = "agora"
        if not ref_data:
            return False
        return ref_data

    def get_tx(self, tx):
        """
        Get the transaction information for a transaction ID.
        :param reference: trade reference
        :type reference: string
        :return: dict of trade information
        :rtype: dict
        """
        tx_data = r.hgetall(f"tx.{tx}")
        tx_data = util.convert(tx_data)
        if not tx_data:
            return False
        return tx_data

    def del_ref(self, reference):
        """
        Delete a given reference from the Redis database.
        :param reference: trade reference to delete
        :type reference: string
        """
        tx = self.ref_to_tx(reference)
        r.delete(f"trade.{reference}")
        r.delete(f"trade.{tx}.reference")

    def cleanup(self, references):
        """
        Reconcile the internal reference database with a given list of references.
        Delete all internal references not present in the list and clean up artifacts.
        :param references: list of references to reconcile against
        :type references: list
        """
        for tx, reference in self.get_ref_map().items():
            if reference not in references:
                self.log.info(f"Archiving trade reference: {reference} / TX: {tx}")
                r.rename(f"trade.{tx}.reference", f"archive.trade.{tx}.reference")
                r.rename(f"trade.{reference}", f"archive.trade.{reference}")

    def tx_to_ref(self, tx):
        """
        Convert a trade ID to a reference.
        :param tx: trade ID
        :type tx: string
        :return: reference
        :rtype: string
        """
        refs = self.get_refs()
        for reference in refs:
            ref_data = util.convert(r.hgetall(f"trade.{reference}"))
            if not ref_data:
                continue
            if ref_data["id"] == tx:
                return reference

    def ref_to_tx(self, reference):
        """
        Convert a reference to a trade ID.
        :param reference: trade reference
        :type reference: string
        :return: trade ID
        :rtype: string
        """
        ref_data = util.convert(r.hgetall(f"trade.{reference}"))
        if not ref_data:
            return False
        return ref_data["id"]

    def get_total_usd(self):
        """
        Get total USD in all our accounts, bank and trading.
        :return: value in USD
        :rtype float:
        """
        total_sinks_usd = self.sinks.get_total_usd()
        agora_wallet_xmr = self.agora.agora.wallet_balance_xmr()
        if not agora_wallet_xmr["success"]:
            return False
        agora_wallet_btc = self.agora.agora.wallet_balance()
        if not agora_wallet_btc["success"]:
            return False
        lbtc_wallet_btc = self.lbtc.lbtc.wallet_balance()
        if not lbtc_wallet_btc["success"]:
            return False
        total_xmr_agora = agora_wallet_xmr["response"]["data"]["total"]["balance"]
        total_btc_agora = agora_wallet_btc["response"]["data"]["total"]["balance"]
        total_btc_lbtc = lbtc_wallet_btc["response"]["data"]["total"]["balance"]
        # Get the XMR -> USD exchange rate
        xmr_usd = self.money.cg.get_price(ids="monero", vs_currencies=["USD"])

        # Get the BTC -> USD exchange rate
        btc_usd = self.money.cg.get_price(ids="bitcoin", vs_currencies=["USD"])

        # Convert the Agora BTC total to USD
        total_usd_agora_btc = float(total_btc_agora) * btc_usd["bitcoin"]["usd"]

        # Convert the LBTC BTC total to USD
        total_usd_lbtc_btc = float(total_btc_lbtc) * btc_usd["bitcoin"]["usd"]

        # Convert the Agora XMR total to USD
        total_usd_agora_xmr = float(total_xmr_agora) * xmr_usd["monero"]["usd"]

        # Add it all up
        total_usd_agora = total_usd_agora_xmr + total_usd_agora_btc
        total_usd_lbtc = total_usd_lbtc_btc
        total_usd = total_usd_agora + total_usd_lbtc + total_sinks_usd
        cast_es = {
            "price_usd": total_usd,
            "total_usd_agora_xmr": total_usd_agora_xmr,
            "total_usd_agora_btc": total_usd_agora_btc,
            "total_usd_lbtc_btc": total_usd_lbtc_btc,
            "total_xmr_agora": total_xmr_agora,
            "total_btc_agora": total_btc_agora,
            "total_btc_lbtc": total_btc_lbtc,
            "xmr_usd": xmr_usd["monero"]["usd"],
            "btc_usd": btc_usd["bitcoin"]["usd"],
            "total_sinks_usd": total_sinks_usd,
            "total_usd_agora": total_usd_agora,
        }
        self.write_to_es("get_total_usd", cast_es)
        return total_usd

    # TODO: possibly refactor this into smaller functions which don't return as much stuff
    # check if this is all really needed in the corresponding withdraw function
    def get_total(self):
        """
        Get all the values corresponding to the amount of money we hold.
        :return: ((total SEK, total USD, total GBP), (total XMR USD, total BTC USD), (total XMR, total BTC))
        :rtype: tuple(tuple(float, float, float), tuple(float, float), tuple(float, float))
        """
        total_sinks_usd = self.sinks.get_total_usd()
        agora_wallet_xmr = self.agora.agora.wallet_balance_xmr()
        if not agora_wallet_xmr["success"]:
            self.log.error("Could not get Agora XMR wallet total.")
            return False
        agora_wallet_btc = self.agora.agora.wallet_balance()
        if not agora_wallet_btc["success"]:
            self.log.error("Could not get Agora BTC wallet total.")
            return False
        lbtc_wallet_btc = self.lbtc.lbtc.wallet_balance()
        if not lbtc_wallet_btc["success"]:
            return False
        total_xmr_agora = agora_wallet_xmr["response"]["data"]["total"]["balance"]
        total_btc_agora = agora_wallet_btc["response"]["data"]["total"]["balance"]
        total_btc_lbtc = lbtc_wallet_btc["response"]["data"]["total"]["balance"]
        # Get the XMR -> USD exchange rate
        xmr_usd = self.money.cg.get_price(ids="monero", vs_currencies=["USD"])

        # Get the BTC -> USD exchange rate
        btc_usd = self.money.cg.get_price(ids="bitcoin", vs_currencies=["USD"])

        # Convert the Agora XMR total to USD
        total_usd_agora_xmr = float(total_xmr_agora) * xmr_usd["monero"]["usd"]

        # Convert the Agora BTC total to USD
        total_usd_agora_btc = float(total_btc_agora) * btc_usd["bitcoin"]["usd"]

        # Convert the LBTC BTC total to USD
        total_usd_lbtc_btc = float(total_btc_lbtc) * btc_usd["bitcoin"]["usd"]

        # Add it all up
        total_usd_agora = total_usd_agora_xmr + total_usd_agora_btc
        total_usd_lbtc = total_usd_lbtc_btc
        total_usd = total_usd_agora + total_usd_lbtc + total_sinks_usd

        total_btc_usd = total_usd_agora_btc + total_usd_lbtc_btc
        total_xmr_usd = total_usd_agora_xmr

        total_xmr = total_xmr_agora
        total_btc = total_btc_lbtc + total_btc_agora

        # Convert the total USD price to GBP and SEK
        rates = self.money.get_rates_all()
        price_sek = rates["SEK"] * total_usd
        price_usd = total_usd
        price_gbp = rates["GBP"] * total_usd

        cast = (
            (
                price_sek,
                price_usd,
                price_gbp,
            ),  # Total prices in our 3 favourite currencies
            (
                total_xmr_usd,
                total_btc_usd,
            ),  # Total USD balance in only Agora
            (total_xmr, total_btc),
        )  # Total XMR and BTC balance in Agora

        cast_es = {
            "price_sek": price_sek,
            "price_usd": price_usd,
            "price_gbp": price_gbp,
            "total_usd_agora_xmr": total_usd_agora_xmr,
            "total_usd_agora_btc": total_usd_agora_btc,
            "total_usd_lbtc_btc": total_usd_lbtc_btc,
            "total_xmr_agora": total_xmr_agora,
            "total_btc_agora": total_btc_agora,
            "total_btc_lbtc": total_btc_lbtc,
            "xmr_usd": xmr_usd["monero"]["usd"],
            "btc_usd": btc_usd["bitcoin"]["usd"],
            "total_sinks_usd": total_sinks_usd,
            "total_usd_agora": total_usd_agora,
        }
        self.write_to_es("get_total", cast_es)
        return cast

    def write_to_es(self, msgtype, cast):
        if settings.ES.Enabled == "1":
            cast["type"] = msgtype
            cast["ts"] = str(datetime.now().isoformat())
            cast["xtype"] = "tx"
            self.es.index(index=settings.ES.Index, document=cast)

    def get_remaining(self):
        """
        Check how much profit we need to make in order to withdraw.
        :return: profit remaining in USD
        :rtype: float
        """
        total_usd = self.get_total_usd()
        if not total_usd:
            return False

        withdraw_threshold = float(settings.Money.BaseUSD) + float(settings.Money.WithdrawLimit)
        remaining = withdraw_threshold - total_usd
        cast_es = {
            "remaining_usd": remaining,
        }
        self.write_to_es("get_remaining", cast_es)
        return remaining

    def open_trades_usd_parse_dash(self, dash, rates):
        cumul_usd = 0
        for contact_id, contact in dash.items():
            # We need created at in order to look up the historical prices
            created_at = contact["data"]["created_at"]

            # Reformat the date how CoinGecko likes
            date_parsed = datetime.strptime(created_at, "%Y-%m-%dT%H:%M:%S.%fZ")
            date_formatted = date_parsed.strftime("%d-%m-%Y")

            # Get the historical rates for the right asset, extract the price
            asset = contact["data"]["advertisement"]["asset"]
            if asset == "XMR":
                amount_crypto = contact["data"]["amount_xmr"]
                history = self.money.cg.get_coin_history_by_id(id="monero", date=date_formatted)
                crypto_usd = float(history["market_data"]["current_price"]["usd"])
            elif asset == "BTC":
                amount_crypto = contact["data"]["amount_btc"]
                history = self.money.cg.get_coin_history_by_id(id="bitcoin", date=date_formatted)
                crypto_usd = float(history["market_data"]["current_price"]["usd"])
            # Convert crypto to fiat
            amount = float(amount_crypto) * crypto_usd
            currency = contact["data"]["currency"]
            if not contact["data"]["is_selling"]:
                continue
            if currency == "USD":
                cumul_usd += float(amount)
            else:
                rate = rates[currency]
                amount_usd = float(amount) / rate
                cumul_usd += amount_usd
        return cumul_usd

    def get_open_trades_usd(self):
        """
        Get total value of open trades in USD.
        :return: total trade value
        :rtype: float
        """
        dash_agora = self.agora.wrap_dashboard()
        if dash_agora is False:
            return False
        dash_lbtc = self.lbtc.wrap_dashboard()
        if dash_lbtc is False:
            return False

        rates = self.money.get_rates_all()
        cumul_usd_agora = self.open_trades_usd_parse_dash(dash_agora, rates)
        cumul_usd_lbtc = self.open_trades_usd_parse_dash(dash_lbtc, rates)
        cumul_usd = cumul_usd_agora + cumul_usd_lbtc

        cast_es = {
            "trades_usd": cumul_usd,
        }
        self.write_to_es("get_open_trades_usd", cast_es)
        return cumul_usd

    def get_total_remaining(self):
        """
        Check how much profit we need to make in order to withdraw, taking into account open trade value.
        :return: profit remaining in USD
        :rtype: float
        """
        total_usd = self.get_total_usd()
        total_trades_usd = self.get_open_trades_usd()
        if not total_usd:
            return False
        total_usd += total_trades_usd
        withdraw_threshold = float(settings.Money.BaseUSD) + float(settings.Money.WithdrawLimit)
        remaining = withdraw_threshold - total_usd

        cast_es = {
            "total_remaining_usd": remaining,
        }
        self.write_to_es("get_total_remaining", cast_es)
        return remaining

    def get_total_with_trades(self):
        total_usd = self.get_total_usd()
        if not total_usd:
            return False
        total_trades_usd = self.get_open_trades_usd()
        total_with_trades = total_usd + total_trades_usd
        cast_es = {
            "total_with_trades": total_with_trades,
        }
        self.write_to_es("get_total_with_trades", cast_es)
        return total_with_trades