pluto/core/clients/platform.py

# Twisted/Klein imports
import asyncio
from abc import ABC
from datetime import datetime, timezone
from random import choices
from string import ascii_uppercase

from django.conf import settings

from core.clients.platforms.api.agoradesk import AgoraDesk
from core.lib import db, notify
from core.lib.antifraud import antifraud
from core.lib.money import money
from core.util import logs

# Other library imports
# from orjson import loads


log = logs.get_logger("platform")


class LocalPlatformClient(ABC):
    """
    Initialise the Local API library for Agora.
    """

    async def connect(self):
        self.api = AgoraDesk(self.instance.token)

    async def call_method(self, method, *args, **kwargs):
        """
        Call a method using the self.api object.
        """
        if hasattr(self.api, method):
            return await getattr(self.api, method)(*args, **kwargs)
        else:
            raise Exception(f"Method {method} not found in {self.name} API.")

    # TODO: do in schedules
    # def setup_loop(self):
    #     """
    #     Set up the LoopingCall to get all active trades and messages.
    #     """
    #     log.debug("Setting up loops.")
    #     self.lc_dash = LoopingCall(self.loop_check)
    #     self.lc_dash.start(int(self.sets.RefreshSec))
    #     if settings.Agora.Cheat == "1":
    #         self.lc_cheat = LoopingCall(self.run_cheat_in_thread)
    #         self.lc_cheat.start(int(self.sets.CheatSec))
    #     log.debug("Finished setting up loops.")

    def map_provider(self, provider, reverse=False):
        provider_map = {"NATIONAL_BANK": "national-bank-transfer"}
        if reverse:
            try:
                return next(
                    key for key, value in provider_map.items() if value == provider
                )
            except StopIteration:
                return False
        else:
            try:
                return provider_map[provider]
            except KeyError:
                return False

    async def got_dashboard(self, dash):
        dash_tmp = await self.wrap_dashboard(dash)
        await self.dashboard_hook(dash_tmp)

    async def wrap_dashboard(self, dash=None):  # backwards compatibility with TX
        if not dash:
            # dash = await self.api.dashboard()
            dash = await self.call("dashboard")

        # if dash["response"] is None:
        #    return False
        dash_tmp = {}
        if not dash:
            return False

        print("DASH", dash)
        if dash["contact_count"] > 0:
            for contact in dash["contact_list"]:
                contact_id = contact["data"]["contact_id"]
                dash_tmp[contact_id] = contact
        return dash_tmp

    async def poll(self):
        """
        Calls hooks to parse dashboard info and get all contact messages.
        """
        dashboard_response = await self.call("dashboard")
        await self.got_dashboard(dashboard_response)

        # Get recent messages
        messages = await self.api.recent_messages()
        await self.got_recent_messages(messages)

    # async def get_dashboard_irc(self):
    #     """
    #     Get dashboard helper for IRC only.
    #     """
    #     dash = await self.wrap_dashboard()
    #     rtrn = []
    #     if dash is False:
    #         return False
    #     for contact_id, contact in dash.items():
    #         reference = db.tx_to_ref(contact_id)
    #         buyer = contact["data"]["buyer"]["username"]
    #         amount = contact["data"]["amount"]
    #         if self.name == "agora":
    #             asset = contact["data"]["advertisement"]["asset"]
    #         elif self.name == "lbtc":
    #             asset = "BTC"
    #         if asset == "XMR":
    #             amount_crypto = contact["data"]["amount_xmr"]
    #         elif asset == "BTC":
    #             amount_crypto = contact["data"]["amount_btc"]
    #         currency = contact["data"]["currency"]
    #         provider = contact["data"]["advertisement"]["payment_method"]
    #         if not contact["data"]["is_selling"]:
    #             continue
    #         rtrn.append(
    #             (
    #                 f"[#] [{reference}] ({self.name}) <{buyer}>"
    #                 f" {amount}{currency} {provider} {amount_crypto}{asset}"
    #             )
    #         )
    #     return rtrn

    async def dashboard_hook(self, dash):
        """
        Get information about our open trades.
        Post new trades to IRC and cache trades for the future.
        """
        current_trades = []
        if not dash:
            return
        if not dash.items():
            return
        for contact_id, contact in dash.items():
            reference = await db.tx_to_ref(str(contact_id))
            if reference:
                current_trades.append(reference)
            buyer = contact["data"]["buyer"]["username"]
            amount = contact["data"]["amount"]
            if self.name == "agora":
                asset = contact["data"]["advertisement"]["asset"]
            elif self.name == "lbtc":
                asset = "BTC"
            provider = contact["data"]["advertisement"]["payment_method"]
            ad_id = provider = contact["data"]["advertisement"]["id"]
            if asset == "XMR":
                amount_crypto = contact["data"]["amount_xmr"]
            elif asset == "BTC":
                amount_crypto = contact["data"]["amount_btc"]
            currency = contact["data"]["currency"]
            if not contact["data"]["is_selling"]:
                continue
            reference = await self.new_trade(
                asset,
                contact_id,
                buyer,
                currency,
                amount,
                amount_crypto,
                provider,
                ad_id,
            )
            if reference:
                if reference not in current_trades:
                    current_trades.append(reference)

        # Purge old trades from cache

        if reference and reference not in current_trades:
            current_trades.append(reference)
        messages = await db.cleanup(self.name, current_trades)
        for message in messages:
            await notify.sendmsg(self.instance.user, message, title="Cleanup")

    async def got_recent_messages(self, messages, send_irc=True):
        """
        Get recent messages.
        """
        messages_tmp = {}
        if not messages:
            return False
        if not messages["success"]:
            return False
        if not messages["response"]:
            return False
        if "data" not in messages["response"]:
            log.error(f"Data not in messages response: {messages['response']}")
            return False
        ref_map = await db.get_ref_map()
        open_tx = ref_map.keys()
        for message in messages["response"]["data"]["message_list"]:
            contact_id = str(message["contact_id"])
            username = message["sender"]["username"]
            msg = message["msg"]
            if contact_id not in open_tx:
                continue
            reference = await db.tx_to_ref(contact_id)
            if reference in messages_tmp:
                messages_tmp[reference].append([username, msg])
            else:
                messages_tmp[reference] = [[username, msg]]

            # Send new messages on IRC
            if send_irc:
                for user, message in messages_tmp[reference][::-1]:
                    if reference in self.instance.last_messages:
                        if (
                            not [user, message]
                            in self.instance.last_messages[reference]
                        ):
                            await notify.sendmsg(
                                self.instance.user,
                                f"[{reference}] ({self.name}) <{user}> {message}",
                                title="New message",
                            )
                            # Append sent messages to last_messages so we don't send
                            # them again
                            self.instance.last_messages[reference].append(
                                [user, message]
                            )
                    else:
                        self.instance.last_messages[reference] = [[user, message]]
                        for x in messages_tmp[reference]:
                            await notify.sendmsg(
                                self.instance.user,
                                f"[{reference}] ({self.name}) <{user}> {message}",
                                title="New message",
                            )

        # Purge old trades from cache
        for ref in list(
            self.instance.last_messages
        ):  # We're removing from the list on the fly
            if ref not in messages_tmp:
                del self.instance.last_messages[ref]

        self.instance.save()

        return messages_tmp

    async def enum_ad_ids(self, page=0):
        if self.name == "lbtc" and page == 0:
            page = 1
        ads = await self.api.ads(page=page)
        # ads = await self.api._api_call(api_method="ads", query_values={"page": page})
        if ads is False:
            return False
        ads_total = []
        if not ads["success"]:
            return False
        for ad in ads["response"]["data"]["ad_list"]:
            ads_total.append(ad["data"]["ad_id"])
        if "pagination" in ads["response"]:
            if "next" in ads["response"]["pagination"]:
                page += 1
                ads_iter = await self.enum_ad_ids(page)
                if ads_iter is None:
                    return False
                if ads_iter is False:
                    return False
                for ad in ads_iter:
                    ads_total.append(ad)
        return ads_total

    async def enum_ads(self, requested_asset=None, page=0):
        if self.name == "lbtc" and page == 0:
            page = 1
        query_values = {"page": page}
        if requested_asset:
            query_values["asset"] = requested_asset
        # ads = await self.api._api_call(api_method="ads", query_values=query_values)
        ads = await self.api.ads(page=page)
        if ads is False:
            return False
        ads_total = []
        if not ads["success"]:
            return False
        if not ads["response"]:
            return False
        for ad in ads["response"]["data"]["ad_list"]:
            if self.name == "agora":
                asset = ad["data"]["asset"]
            elif self.name == "lbtc":
                asset = "BTC"
            ad_id = ad["data"]["ad_id"]
            country = ad["data"]["countrycode"]
            currency = ad["data"]["currency"]
            provider = ad["data"]["online_provider"]
            ads_total.append([asset, ad_id, country, currency, provider])
        if "pagination" in ads["response"]:
            if "next" in ads["response"]["pagination"]:
                page += 1
                ads_iter = await self.enum_ads(requested_asset, page)
                if ads_iter is None:
                    return False
                if ads_iter is False:
                    return False
                for ad in ads_iter:
                    ads_total.append([ad[0], ad[1], ad[2], ad[3], ad[4]])
        return ads_total

    def last_online_recent(self, date):
        """
        Check if the last online date was recent.
        :param date: date last online
        :type date: string
        :return: bool indicating whether the date was recent enough
        :rtype: bool
        """
        if "+" in date:
            # for LBTC
            # 2022-04-16T08:53:58+00:00
            date_split = date.split("+")
            date_split[1].replace(".", "")
            date_split[1].replace(":", "")
            date = "+".join(date_split)
            date_string = "%Y-%m-%dT%H:%M:%S%z"
            now = datetime.now(timezone.utc)
        else:
            date_string = "%Y-%m-%dT%H:%M:%S.%fZ"
            now = datetime.now()
        date_parsed = datetime.strptime(date, date_string)
        sec_ago_date = (now - date_parsed).total_seconds()
        return sec_ago_date < 172800

    async def enum_public_ads(self, asset, currency, providers=None, page=0):
        if self.name == "lbtc" and page == 0:
            page = 1
        to_return = []
        # if asset == "XMR":
        #    coin = "monero"
        # elif asset == "BTC":
        #    coin = "bitcoins"
        if not providers:
            providers = ["NATIONAL_BANK"]
        # buy-monero-online, buy-bitcoin-online
        # Work around Agora weirdness calling it bitcoins
        # ads = await self.api._api_call(
        #    api_method=f"buy-{coin}-online/{currency}",
        #    query_values={"page": page},
        # )
        if asset == "XMR":
            ads = await self.api.buy_monero_online(currency_code=currency, page=page)
        elif asset == "BTC":
            ads = await self.api.buy_bitcoins_online(currency_code=currency, page=page)
        # with open("pub.json", "a") as f:
        #   import json
        #   f.write(json.dumps([page, currency, asset, ads])+"\n")
        # f.close()
        if ads is None:
            return False
        if ads is False:
            return False
        if ads["response"] is None:
            return False
        if "data" not in ads["response"]:
            return False
        for ad in ads["response"]["data"]["ad_list"]:
            provider = ad["data"]["online_provider"]
            if self.name == "lbtc":
                provider_test = self.map_provider(provider)
            else:
                provider_test = provider
            if provider_test not in providers:
                continue
            date_last_seen = ad["data"]["profile"]["last_online"]
            # Check if this person was seen recently
            if not self.last_online_recent(date_last_seen):
                continue
            ad_id = str(ad["data"]["ad_id"])
            username = ad["data"]["profile"]["username"]
            temp_price = ad["data"]["temp_price"]
            if ad["data"]["currency"] != currency:
                continue
            to_append = [ad_id, username, temp_price, provider, asset, currency]
            if to_append not in to_return:
                to_return.append(to_append)
            # await [ad_id, username, temp_price, provider, asset, currency]
        if "pagination" in ads["response"]:
            if "next" in ads["response"]["pagination"]:
                page += 1
                ads_iter = await self.enum_public_ads(asset, currency, providers, page)
                if ads_iter is None:
                    return False
                if ads_iter is False:
                    return False
                for ad in ads_iter:
                    to_append = [ad[0], ad[1], ad[2], ad[3], ad[4], ad[5]]
                    if to_append not in to_return:
                        to_return.append(to_append)
        return to_return

    async def run_cheat_in_thread(self, assets=None):
        """
        Update prices in another thread.
        """
        if not assets:
            all_assets = ["XMR"]
            assets_not_run = set(all_assets) ^ set(self.cheat_run_on)
            if not assets_not_run:
                self.cheat_run_on = []
                asset = list(all_assets).pop()
                self.cheat_run_on.append(asset)
            else:
                asset = assets_not_run.pop()
                self.cheat_run_on.append(asset)
            await self.update_prices([asset])
            return asset
        else:
            # deferToThread(self.update_prices, assets)
            await self.update_prices(assets)

    async def update_prices(self, assets=None):
        # Get all public ads for the given assets
        public_ads = await self.get_all_public_ads(assets)
        if not public_ads:
            return False

        # Get the ads to update
        to_update = self.get_new_ad_equations(self.name, public_ads, assets)
        await self.slow_ad_update(to_update)

    async def get_all_public_ads(self, assets=None, currencies=None, providers=None):
        """
        Get all public ads for our listed currencies.
        :return: dict of public ads keyed by currency
        :rtype: dict
        """
        public_ads = {}
        crypto_map = {
            "XMR": "monero",
            "BTC": "bitcoin",
        }

        assets = assets or self.instance.ads_assets
        # Get all currencies we have ads for, deduplicated

        providers = providers or self.instance.ads_providers
        currencies = currencies or self.instance.currencies
        # We want to get the ads for each of these currencies and return the result
        rates = await money.cg.get_price(
            ids=["monero", "bitcoin"], vs_currencies=currencies
        )
        for asset in assets:
            for currency in currencies:
                cg_asset_name = crypto_map[asset]
                try:
                    rates[cg_asset_name][currency.lower()]
                except KeyError:
                    log.debug(f"Error getting public ads for currency: {currency}")
                    continue
                ads_list = await self.enum_public_ads(asset, currency, providers)
                if not ads_list:
                    log.debug("Error getting ads list.")
                    continue
                ads = await money.lookup_rates(self.name, ads_list, rates=rates)
                if not ads:
                    log.debug("Error lookup up rates.")
                    continue
                log.debug("Writing to ES.")
                await self.write_to_es_ads("ads", ads)
                if currency in public_ads:
                    for ad in list(ads):
                        if ad not in public_ads[currency]:
                            public_ads[currency].append(ad)
                else:
                    public_ads[currency] = ads

        return public_ads

    async def write_to_es_ads(self, msgtype, ads):
        for ad in ads:
            cast = {
                "id": ad[0],
                "username": ad[1],
                "price": ad[2],
                "provider": ad[3],
                "asset": ad[4],
                "currency": ad[5],
                "margin": ad[6],
                "ts": str(datetime.now().isoformat()),
                "xtype": msgtype,
                "market": self.name,
                "type": "platform",
                "user_id": self.instance.user.id,
                "platform_id": self.instance.id,
            }
            await money.es.index(index=settings.ELASTICSEARCH_INDEX_ADS, body=cast)

    async def slow_ad_update(self, ads):
        """
        Slow ad equation update utilising exponential backoff in order to guarantee all
         ads are updated.
        :param ads: our list of ads
        """
        iterations = 0
        throttled = 0
        assets = set()
        currencies = set()
        while not all([x[4] for x in ads]) or iterations == 1000:
            for ad_index in range(len(ads)):
                ad_id, new_formula, asset, currency, actioned = ads[ad_index]
                assets.add(asset)
                currencies.add(currency)
                if not actioned:
                    rtrn = await self.api.ad_equation(ad_id, new_formula)
                    if rtrn["success"]:
                        ads[ad_index][4] = True
                        throttled = 0
                        continue
                    else:
                        if "error_code" not in rtrn["response"]["error"]:
                            log.error(
                                (
                                    f"Error code not in return for ad {ad_id}: "
                                    f"{rtrn['response']}"
                                )
                            )
                            return
                        if rtrn["response"]["error"]["error_code"] == 429:
                            throttled += 1
                            sleep_time = pow(throttled, 1.9)
                            log.info(
                                (
                                    f"Throttled {throttled} times while updating "
                                    f"{ad_id}, "
                                    f"sleeping for {sleep_time} seconds"
                                )
                            )
                            # We're running in a thread, so this is fine
                            await asyncio.sleep(sleep_time)
                        log.error(f"Error updating ad {ad_id}: {rtrn['response']}")
                        continue
            iterations += 1

    async def nuke_ads(self):
        """
        Delete all of our adverts.
        :return: True or False
        :rtype: bool
        """
        ads = await self.enum_ad_ids()
        return_ids = []
        if ads is False:
            return False
        for ad_id in ads:
            rtrn = await self.api.ad_delete(ad_id)
            return_ids.append(rtrn["success"])
        return all(return_ids)

    async def create_ad(
        self,
        asset,
        countrycode,
        currency,
        provider,
        payment_details,
        ad,
        visible=None,
        edit=False,
        ad_id=None,
    ):
        """
        Post an ad with the given asset in a country with a given currency.
        Convert the min and max amounts from settings to the given currency with
        CurrencyRates.
        :param asset: the crypto asset to list (XMR or BTC)
        :type asset: string
        :param countrycode: country code
        :param currency: currency code
        :param payment_details: the payment details
        :type countrycode: string
        :type currency: string
        :type payment_details: dict
        :return: data about created object or error
        :rtype: dict
        """

        if payment_details:
            payment_details_text = self.format_payment_details(
                currency,
                payment_details,
                ad,
            )
            ad_text = self.format_ad(asset, currency, payment_details_text, ad)
            min_amount, max_amount = await money.get_minmax(
                self.instance.min_trade_size_usd,
                self.instance.max_trade_size_usd,
                asset,
                currency,
            )
            if self.name == "lbtc":
                bank_name = payment_details["bank"]

        # if self.name == "agora":
        price_formula = (
            f"coingecko{asset.lower()}usd*"
            f"usd{currency.lower()}*{self.instance.margin}"
        )
        # elif self.name == "lbtc":
        #     price_formula = f"btc_in_usd*{self.instance.margin}*USD_in_{currency}"

        form = {
            "country_code": countrycode,
            "currency": currency,
            "trade_type": "ONLINE_SELL",
            # "asset": asset,
            "price_equation": price_formula,
            "track_max_amount": False,
            "require_trusted_by_advertiser": False,
            "online_provider": provider,
            "require_feedback_score": 0,
        }
        if self.name == "agora":
            form["asset"] = asset
            form["payment_method_details"] = ad.payment_method_details
            form["online_provider"] = provider
        elif self.name == "lbtc":
            form["online_provider"] = self.map_provider(provider, reverse=True)
        if visible is False:
            form["visible"] = False
        elif visible is True:
            form["visible"] = True
        if payment_details:
            form["account_info"] = payment_details_text
            form["msg"] = ad_text
            form["min_amount"] = round(min_amount, 2)
            form["max_amount"] = round(max_amount, 2)
            if self.name == "lbtc":
                form["bank_name"] = bank_name

        if edit:
            ad_response = await self.api.ad(ad_id=ad_id, **form)
            if ad_response["success"]:
                self.instance.platform_ad_ids[ad_id] = str(ad.id)
            self.instance.save()
        else:
            ad_response = await self.api.ad_create(**form)
            if ad_response["success"]:
                ad_id = ad_response["response"]["data"]["ad_id"]
                self.instance.platform_ad_ids[ad_id] = str(ad.id)
            self.instance.save()

        print("AD", ad_response)
        return ad_response

    async def dist_countries(self, ad, filter_asset=None):
        """
        Distribute our advert into all countries and providers listed in the config.
        Exits on errors.
        :return: False or dict with response
        :rtype: bool or dict
        """
        dist_list = list(self.create_distribution_list(ad, filter_asset))
        our_ads = await self.enum_ads()
        (
            supported_currencies,
            account_info,
        ) = self.get_valid_account_details(ad)
        # Let's get rid of the ad IDs and make it a tuple like dist_list
        our_ads = [(x[0], x[2], x[3], x[4]) for x in our_ads]
        if not our_ads:
            log.error("Could not get our ads.")
            return False
        to_return = []
        for asset, countrycode, currency, provider in dist_list:
            if (asset, countrycode, currency, provider) not in our_ads:
                if currency in supported_currencies:
                    # Create the actual ad and pass in all the stuff
                    rtrn = await self.create_ad(
                        asset,
                        countrycode,
                        currency,
                        provider,
                        payment_details=account_info[currency],
                        ad=ad,
                    )
                    # Bail on first error, let's not continue
                    if rtrn is False:
                        return False
                    to_return.append(rtrn)
        return to_return

    async def redist_countries(self, ad):
        """
        Redistribute our advert details into all our listed adverts.
        This will edit all ads and update the details. Only works if we have already
        run dist.
        This will not post any new ads.
        Exits on errors.
        :return: False or dict with response
        :rtype: bool or dict
        """
        our_ads = await self.enum_ads()
        (
            supported_currencies,
            account_info,
        ) = self.get_valid_account_details(ad)
        if not our_ads:
            log.error("Could not get our ads.")
            return False
        to_return = []
        for asset, ad_id, countrycode, currency, provider in our_ads:
            if asset not in self.instance.ads_assets:
                continue
            if provider not in self.instance.ads_providers:
                continue
            if currency in supported_currencies:
                rtrn = await self.create_ad(
                    asset,
                    countrycode,
                    currency,
                    provider,
                    payment_details=account_info[currency],
                    ad=ad,
                    visible=ad.visible,
                    edit=True,
                    ad_id=ad_id,
                )
                # Bail on first error, let's not continue
                if rtrn is False:
                    return False
                to_return.append((rtrn, ad_id))
        return to_return

    async def strip_duplicate_ads(self):
        """
        Remove duplicate ads.
        :return: list of duplicate ads
        :rtype: list
        """
        existing_ads = await self.enum_ads()
        _size = len(existing_ads)
        repeated = []
        for i in range(_size):
            k = i + 1
            for j in range(k, _size):
                if (
                    existing_ads[i] == existing_ads[j]
                    and existing_ads[i] not in repeated
                ):
                    repeated.append(existing_ads[i])

        actioned = []
        for ad_id, country, currency in repeated:
            rtrn = await self.api.ad_delete(ad_id)
            actioned.append(rtrn["success"])

        return all(actioned)

    async def release_funds(self, trade_id, reference):
        stored_trade = await db.get_ref(reference)
        logmessage = f"All checks passed, releasing funds for {trade_id} {reference}"
        log.info(logmessage)
        title = "Releasing escrow"
        await notify.sendmsg(self.instance.user, logmessage, title=title)
        release = self.release_funds
        post_message = self.api.contact_message_post

        rtrn = await release(trade_id)
        if rtrn["message"] == "OK":
            post_message(trade_id, "Thanks! Releasing now :)")
            return True
        else:
            logmessage = f"Release funds unsuccessful: {rtrn['message']}"
            title = "Release unsuccessful"
            log.error(logmessage)
            await notify.sendmsg(self.instance.user, logmessage, title=title)
            return

        # Parse the escrow release response
        message = rtrn["message"]
        # message_long = rtrn["response"]["data"]["message"]
        self.irc.sendmsg(f"{dumps(message)}")

    async def update_trade_tx(self, reference, txid):
        """
        Update a trade to point to a given transaction ID.
        Return False if the trade already has a mapped transaction.
        """
        existing_tx = await db.r.hget(f"trade.{reference}", "tx")
        if existing_tx is None:
            return None
        elif existing_tx == b"":
            await db.r.hset(f"trade.{reference}", "tx", txid)
            return True
        else:  # Already a mapped transaction
            return False

    async def release_map_trade(self, reference, tx):
        """
        Map a trade to a transaction and release if no other TX is
        mapped to the same trade.
        """
        stored_trade = await db.get_ref(reference)
        if not stored_trade:
            log.error(f"Could not get stored trade for {reference}.")
            return None
        tx_obj = await db.get_tx(tx)
        if not tx_obj:
            log.error(f"Could not get TX for {tx}.")
            return None
        platform_buyer = stored_trade["buyer"]
        bank_sender = tx_obj["sender"]
        trade_id = stored_trade["id"]
        is_updated = await self.update_trade_tx(reference, tx)
        if is_updated is None:
            return None
        elif is_updated is True:
            # We mapped the trade successfully
            self.release_funds(trade_id, reference)
            antifraud.add_bank_sender(platform_buyer, bank_sender)
            return True
        elif is_updated is False:
            # Already mapped
            log.error(f"Trade {reference} already has a TX mapped, cannot map {tx}.")
        return False

    async def new_trade(
        self,
        asset,
        trade_id,
        buyer,
        currency,
        amount,
        amount_crypto,
        provider,
        ad_id,
    ):
        """
        Called when we have a new trade in Agora.
        Store details in Redis, generate a reference and optionally let the customer
        know the reference.
        """
        reference = "".join(choices(ascii_uppercase, k=5))
        reference = f"AGR-{reference}"
        existing_ref = await db.r.get(f"trade.{trade_id}.reference")
        if not existing_ref:
            to_store = {
                "id": trade_id,
                "tx": "",
                "asset": asset,
                "buyer": buyer,
                "currency": currency,
                "amount": amount,
                "amount_crypto": amount_crypto,
                "reference": reference,
                "provider": provider,
            }
            log.info(f"Storing trade information: {str(to_store)}")
            await db.r.hmset(f"trade.{reference}", to_store)
            await db.r.set(f"trade.{trade_id}.reference", reference)
            message = f"Generated reference for {trade_id}: {reference}"
            title = "Generated reference"
            await notify.sendmsg(self.instance.user, message, title=title)
            # uid = self.ux.verify.create_uid(subclass, buyer)
            # verified = self.ux.verify.get_external_user_id_status(uid)
            # if verified != "GREEN":
            #     log.info(f"UID {uid} is not verified, sending link.")
            #     self.antifraud.send_verification_url(subclass, uid, trade_id)
            # else:  # User is verified
            #     log.info(f"UID {uid} is verified.")
            ad_obj = self.instance.get_ad(ad_id)
            if not ad_obj:
                log.error(f"Could not get ad object for {ad_id}.")
                return

            await self.send_bank_details(currency, trade_id, ad_obj)
            await self.send_reference(trade_id, reference)
        if existing_ref:
            return db.convert(existing_ref)
        else:
            return reference

    def get_uid(self, external_user_id):
        """
        Get the platform and username from the external user ID.
        """
        spl = external_user_id.split("|")
        if not len(spl) == 2:
            log.error(f"Split invalid, cannot get customer: {spl}")
            return False
        platform, username = spl
        return (platform, username)

    async def find_trades_by_uid(self, uid):
        """
        Find a list of trade IDs and references by a customer UID.
        :return: tuple of (platform, trade_id, reference, currency)
        """
        platform, username = self.get_uid(uid)
        refs = await db.get_refs()
        matching_trades = []
        for reference in refs:
            ref_data = await db.get_ref(reference)
            tx_username = ref_data["buyer"]
            trade_id = ref_data["id"]
            currency = ref_data["currency"]
            if tx_username == username:
                to_append = (platform, trade_id, reference, currency)
                matching_trades.append(to_append)
        return matching_trades

    def get_send_settings(self, platform):
        if platform == "agora":
            send_setting = self.instance.send
            post_message = self.api.contact_message_post

        return (send_setting, post_message)

    async def send_reference(self, trade_id, reference):
        """
        Send the reference to a customer.
        """
        if self.instance.send is True:
            print("SEND IS TRUE REF")
            await self.api.contact_message_post(
                trade_id,
                f"When sending the payment please use reference code: {reference}",
            )

    async def send_bank_details(self, currency, trade_id, ad):
        """
        Send the bank details to a trade.
        """
        log.info(f"Sending bank details/reference for {trade_id}")
        if self.instance.send is True:
            print("SEND IS TRUE")
            account_info = self.get_matching_account_details(currency, ad)
            print("ACCOUNT INFO", account_info)
            formatted_account_info = self.format_payment_details(
                currency, account_info, ad, real=True
            )
            print("formatted_account_info", formatted_account_info)
            if not formatted_account_info:
                log.error(f"Payment info invalid: {formatted_account_info}")
                return
            await self.api.contact_message_post(
                trade_id,
                f"Payment details: \n{formatted_account_info}",
            )

    def get_all_assets(self, platform):
        raise Exception

    def get_all_providers(self, platform):
        raise Exception

    def get_all_currencies(self, platform):
        raise Exception

    def get_new_ad_equations(self, platform, public_ads, assets=None):
        """
        Update all our prices.
        :param public_ads: dictionary of public ads keyed by currency
        :type public_ads: dict
        :return: list of ads to modify
        :rtype: list
        """
        username = self.instance.username
        min_margin = self.instance.min_margin
        max_margin = self.instance.max_margin

        to_update = []

        # NOTES:
        # Get all ads for each currency, with all the payment methods.
        # Create a function to, in turn, filter these so it contains only one payment
        # method. Run autoprice on this.
        # Append all results to to_update. Repeat for remaining payment methods, then
        # call slow update.

        # (asset, currency, provider)
        assets = self.instance.ads_assets
        currencies = self.currencies
        providers = self.instance.ads_providers
        # if platform == "lbtc":
        #     providers = [
        #         self.sources.lbtc.map_provider(x, reverse=True) for x in providers
        #     ]

        brute = [
            (asset, currency, provider)
            for asset in assets
            for currency in currencies
            for provider in providers
        ]
        for asset, currency, provider in brute:
            # Filter currency
            try:
                public_ads_currency = public_ads[currency]
            except KeyError:
                if currency == "GBP":
                    log.error("Error getting public ads for currency GBP, aborting")
                    break
                continue
            # Filter asset
            public_ads_filtered = [ad for ad in public_ads_currency if ad[4] == asset]

            # Filter provider
            public_ads_filtered = [
                ad for ad in public_ads_filtered if ad[3] == provider
            ]
            our_ads = [ad for ad in public_ads_filtered if ad[1] == username]
            if not our_ads:
                log.warning(
                    (
                        f"No ads found in {platform} public listing for "
                        f"{asset} {currency}"
                        f" {provider}"
                    )
                )
                continue
            new_margin = self.autoprice(
                username, min_margin, max_margin, public_ads_filtered, currency
            )
            if platform == "agora":
                new_formula = (
                    f"coingecko{asset.lower()}usd*usd{currency.lower()}*{new_margin}"
                )
            elif platform == "lbtc":
                new_formula = f"btc_in_usd*{new_margin}*USD_in_{currency}"
            for ad in our_ads:
                ad_id = ad[0]
                asset = ad[4]
                our_margin = ad[5]
                if new_margin != our_margin:
                    to_update.append([str(ad_id), new_formula, asset, currency, False])

        return to_update

    def autoprice(self, username, min_margin, max_margin, ads, currency):
        """
        Helper function to automatically adjust the price up/down in certain markets
        in order to gain the most profits and sales.
        :param ads: list of ads
        :type ads: list of lists
        :param currency: currency of the ads
        :type currency: string
        :return: the rate we should use for this currency
        :rtype: float
        """
        # log.debug("Autoprice starting for {x}", x=currency)
        # Find cheapest ad
        # Filter by 3rd index on each ad list to find the cheapest
        min_margin_ad = min(ads, key=lambda x: x[6])
        # log.debug("Minimum margin ad: {x}", x=min_margin_ad)

        # Find second cheapest that is not us
        # Remove results from ads that are us
        ads_without_us = [ad for ad in ads if not ad[1] == username]
        # log.debug("Ads without us: {x}", x=ads_without_us)
        # Find ads above our min that are not us
        ads_above_our_min_not_us = [
            ad for ad in ads_without_us if ad[6] > float(min_margin)
        ]
        # log.debug("Ads above our min not us: {x}", x=ads_above_our_min_not_us)
        # Check that this list without us is not empty
        if ads_without_us:
            # Find the cheapest from these
            min_margin_ad_not_us = min(ads_without_us, key=lambda x: x[6])
            # log.debug("Min margin ad not us: {x}", x=min_margin_ad_not_us)
            # Lowball the lowest ad that is not ours
            lowball_lowest_not_ours = min_margin_ad_not_us[6]  # - 0.005
            # log.debug("Lowball lowest not ours: {x}", x=lowball_lowest_not_ours)

        # Check if the username field of the cheapest ad matches ours
        if min_margin_ad[1] == username:
            # log.debug("We are the cheapest for: {x}", x=currency)
            # We are the cheapest!
            # Are all of the ads ours?
            all_ads_ours = all([ad[1] == username for ad in ads])
            if all_ads_ours:
                # log.debug("All ads are ours for: {x}", x=currency)
                # Now we know it's safe to return the maximum value
                return float(max_margin)
            else:
                # log.debug("All ads are NOT ours for: {x}", x=currency)
                # All the ads are not ours, but we are first...
                # Check if the lowballed, lowest (that is not ours) ad's margin
                # is less than our minimum
                if lowball_lowest_not_ours < float(min_margin):
                    # log.debug("Lowball lowest not ours less than MinMargin")
                    return float(min_margin)
                elif lowball_lowest_not_ours > float(max_margin):
                    # log.debug("Lowball lowest not ours more than MaxMargin")
                    return float(max_margin)
                else:
                    # log.debug("Returning lowballed figure:
                    #  {x}", x=lowball_lowest_not_ours)
                    return lowball_lowest_not_ours
        else:
            # log.debug("We are NOT the cheapest for: {x}", x=currency)
            # We are not the cheapest :(
            # Check if this list is empty
            if not ads_above_our_min_not_us:
                # Return the maximum margin?
                return float(max_margin)
            # Find cheapest ad above our min that is not us
            cheapest_ad = min(ads_above_our_min_not_us, key=lambda x: x[4])
            cheapest_ad_margin = cheapest_ad[6]  # - 0.005
            if cheapest_ad_margin > float(max_margin):
                # log.debug("Cheapest ad not ours more than MaxMargin")
                return float(max_margin)
            # log.debug("Cheapest ad above our min that is not us: {x}", x=cheapest_ad)
            return cheapest_ad_margin

    def create_distribution_list(self, ad, filter_asset=None):
        """
        Create a list for distribution of ads.
        :return: generator of asset, countrycode, currency, provider
        :rtype: generator of tuples
        """

        dist_list_raw = ad.dist_list
        dist_list_lines = dist_list_raw.splitlines()

        distlist = []
        for line in dist_list_lines:
            l_currency, l_country = line.split()
            distlist.append([l_currency, l_country])

        # Iterate providers like REVOLUT, NATIONAL_BANK
        for provider in self.instance.ads_providers:
            # Iterate assets like XMR, BTC
            for asset in self.instance.ads_assets:
                # Iterate pairs of currency and country like EUR, GB
                for currency, countrycode in distlist:
                    if filter_asset:
                        if asset == filter_asset:
                            yield (asset, countrycode, currency, provider)
                    else:
                        yield (asset, countrycode, currency, provider)

    def get_valid_account_details(self, ad):
        currencies = self.instance.currencies
        account_info = self.instance.account_info
        print("GET VALID", ad)
        account_whitelist = ad.account_whitelist
        if account_whitelist:
            whitelist = account_whitelist.splitlines()
        else:
            whitelist = None
        print("WHITELIST", whitelist)
        currency_account_info_map = {}
        for currency in currencies:
            for bank, accounts in account_info.items():
                for account in accounts:
                    if account["currency"] == currency:
                        if whitelist:
                            if account["account_id"] not in whitelist:
                                continue
                        currency_account_info_map[currency] = account["account_number"]
                        currency_account_info_map[currency]["bank"] = bank.split("_")[0]
                        currency_account_info_map[currency]["recipient"] = account[
                            "ownerName"
                        ]
        return (currencies, currency_account_info_map)

    def get_matching_account_details(self, currency, ad):
        (
            supported_currencies,
            currency_account_info_map,
        ) = self.get_valid_account_details(ad)
        if currency not in supported_currencies:
            return False
        return currency_account_info_map[currency]

    # def _distribute_account_details(self, platform, currencies=None, account_info=None):
    #     """
    #     Distribute account details for ads.
    #     We will disable ads we can't support.
    #     """

    #     if not currencies:
    #         currencies = self.instance.currencies
    #     if not account_info:
    #         account_info = self.instance.account_info
    #     (
    #         supported_currencies,
    #         currency_account_info_map,
    #     ) = self.get_valid_account_details(platform)

    #     # not_supported = [currency for currency in all_currencies if
    #     # currency not in supported_currencies]

    #     our_ads = self.enum_ads()

    #     supported_ads = [ad for ad in our_ads if ad[3] in supported_currencies]

    #     not_supported_ads = [ad for ad in our_ads if ad[3] not in supported_currencies]

    #     for ad in supported_ads:
    #         asset = ad[0]
    #         countrycode = ad[2]
    #         currency = ad[3]
    #         provider = ad[4]
    #         payment_details = currency_account_info_map[currency]
    #         ad_id = ad[1]
    #         self.create_ad(
    #             asset,
    #             countrycode,
    #             currency,
    #             provider,
    #             payment_details,
    #             visible=True,
    #             edit=True,
    #             ad_id=ad_id,
    #         )

    #     for ad in not_supported_ads:
    #         asset = ad[0]
    #         countrycode = ad[2]
    #         currency = ad[3]
    #         provider = ad[4]
    #         ad_id = ad[1]
    #         self.create_ad(
    #             asset,
    #             countrycode,
    #             currency,
    #             provider,
    #             payment_details=False,
    #             visible=False,
    #             edit=True,
    #             ad_id=ad_id,
    #         )

    # def distribute_account_details(self, currencies=None, account_info=None):
    #     """
    #     Helper to distribute the account details for all platforms.
    #     """
    #     platforms = "agora"
    #     for platform in platforms:
    #         self._distribute_account_details(
    #             platform, currencies=currencies, account_info=account_info
    # )

    def format_ad(self, asset, currency, payment_details_text, ad):
        """
        Format the ad.
        """
        ad_text = ad.text

        # Substitute the currency
        ad_text = ad_text.replace("$CURRENCY$", currency)

        # Substitute the asset
        ad_text = ad_text.replace("$ASSET$", asset)

        # Substitute the payment details
        ad_text = ad_text.replace("$PAYMENT$", payment_details_text)

        # Strip extra tabs
        ad_text = ad_text.replace("\\t", "\t")
        return ad_text

    def format_payment_details(self, currency, payment_details, ad, real=False):
        """
        Format the payment details.
        """
        if not payment_details:
            return False
        if real:
            payment = ad.payment_details_real
        else:
            payment = ad.payment_details

        payment_text = ""
        for field, value in payment_details.items():
            formatted_name = field.replace("_", " ")
            formatted_name = formatted_name.capitalize()
            payment_text += f"* {formatted_name}: **{value}**"
            if field != list(payment_details.keys())[-1]:  # No trailing newline
                payment_text += "\n"

        payment = payment.replace("$PAYMENT$", payment_text)
        payment = payment.replace("$CURRENCY$", currency)

        return payment