pluto/handler/agora.py

# Twisted/Klein imports
from twisted.logger import Logger
from twisted.internet.task import LoopingCall
from twisted.internet.threads import deferToThread

# Other library imports
from json import loads
from forex_python.converter import CurrencyRates
from agoradesk_py import AgoraDesk
from httpx import ReadTimeout, ReadError
from pycoingecko import CoinGeckoAPI
from datetime import datetime
from time import sleep
from pyotp import TOTP

# Project imports
from settings import settings

log = Logger("agora.global")


def handle_exceptions(func):
    def inner_function(*args, **kwargs):
        try:
            rtrn = func(*args, **kwargs)
        except (ReadTimeout, ReadError):
            return False
        if isinstance(rtrn, dict):
            if "success" in rtrn:
                if "message" in rtrn:
                    if not rtrn["success"] and rtrn["message"] == "API ERROR":
                        if "error_code" in rtrn["response"]["error"]:
                            log.error("API error: {code}", code=rtrn["response"]["error"]["error_code"])
                            return False
                        else:
                            log.error("API error: {code}", code=rtrn["response"]["error"])
                            return False
        return rtrn

    return inner_function


class Agora(object):
    """
    AgoraDesk API handler.
    """

    def __init__(self):
        """
        Initialise the AgoraDesk and CurrencyRates APIs.
        Initialise the last_dash storage for detecting new trades.
        """
        self.log = Logger("agora")
        self.agora = AgoraDesk(settings.Agora.Token)
        self.cr = CurrencyRates()
        self.cg = CoinGeckoAPI()

        # Cache for detecting new trades
        self.last_dash = set()

        # Cache for detecting new messages
        self.last_messages = {}

        # Assets that cheat has been run on
        self.cheat_run_on = []

    def set_irc(self, irc):
        self.irc = irc

    def set_tx(self, tx):
        self.tx = tx

    def set_notify(self, notify):
        self.notify = notify

    def setup_loop(self):
        """
        Set up the LoopingCall to get all active trades and messages.
        """
        self.lc_dash = LoopingCall(self.loop_check)
        self.lc_dash.start(int(settings.Agora.RefreshSec))
        if settings.Agora.Cheat == "1":
            self.lc_cheat = LoopingCall(self._update_prices, None, None)
            self.lc_cheat.start(int(settings.Agora.CheatSec))

    @handle_exceptions
    def wrap_dashboard(self):
        dash = self.agora.dashboard_seller()
        if dash is None:
            return False
        if dash is False:
            return False
        if dash["response"] is None:
            return False
        dash_tmp = {}
        if not dash.items():
            return False
        if "data" not in dash["response"].keys():
            self.log.error("Data not in dashboard response: {content}", content=dash)
            return dash_tmp
        if dash["response"]["data"]["contact_count"] > 0:
            for contact in dash["response"]["data"]["contact_list"]:
                contact_id = contact["data"]["contact_id"]
                dash_tmp[contact_id] = contact
        return dash_tmp

    def loop_check(self):
        """
        Calls hooks to parse dashboard info and get all contact messages.
        """
        dash_tmp = self.wrap_dashboard()

        # Call dashboard hooks
        self.dashboard_hook(dash_tmp)

        # Get recent messages
        self.get_recent_messages()
        return dash_tmp

    def get_dashboard(self):
        """
        Get dashboard helper for IRC only.
        """
        dash = self.wrap_dashboard()
        rtrn = []
        if dash is False:
            return False
        for contact_id, contact in dash.items():
            reference = self.tx.tx_to_ref(contact_id)
            buyer = contact["data"]["buyer"]["username"]
            amount = contact["data"]["amount"]
            asset = contact["data"]["advertisement"]["asset"]
            if asset == "XMR":
                amount_crypto = contact["data"]["amount_xmr"]
            elif asset == "BTC":
                amount_crypto = contact["data"]["amount_btc"]
            currency = contact["data"]["currency"]
            if not contact["data"]["is_selling"]:
                continue
            rtrn.append(f"{reference}: {buyer} {amount}{currency} {amount_crypto}{asset}")
        return rtrn

    def dashboard_hook(self, dash):
        """
        Get information about our open trades.
        Post new trades to IRC and cache trades for the future.
        """
        current_trades = []
        if not dash:
            return
        if not dash.items():
            return
        for contact_id, contact in dash.items():
            reference = self.tx.tx_to_ref(contact_id)
            if reference:
                current_trades.append(reference)
            buyer = contact["data"]["buyer"]["username"]
            amount = contact["data"]["amount"]
            asset = contact["data"]["advertisement"]["asset"]
            if asset == "XMR":
                amount_crypto = contact["data"]["amount_xmr"]
            elif asset == "BTC":
                amount_crypto = contact["data"]["amount_btc"]
            currency = contact["data"]["currency"]
            if not contact["data"]["is_selling"]:
                continue
            if reference not in self.last_dash:
                reference = self.tx.new_trade(asset, contact_id, buyer, currency, amount, amount_crypto)
                if reference:
                    if reference not in current_trades:
                        current_trades.append(reference)
                # Let us know there is a new trade
                self.irc.sendmsg(f"AUTO {reference}: {buyer} {amount}{currency} {amount_crypto}{asset}")
            # Note that we have seen this reference
            self.last_dash.add(reference)

        # Purge old trades from cache
        for ref in list(self.last_dash):  # We're removing from the list on the fly
            if ref not in current_trades:
                self.last_dash.remove(ref)
        if reference and reference not in current_trades:
            current_trades.append(reference)
        self.tx.cleanup(current_trades)

    @handle_exceptions
    def dashboard_release_urls(self):
        """
        Get information about our open trades.
        Post new trades to IRC and cache trades for the future.
        :return: human readable list of strings about our trades or False
        :rtype: list or bool
        """
        dash = self.agora.dashboard_seller()
        if dash is False:
            return False
        dash_tmp = []
        if "data" not in dash["response"]:
            self.log.error("Data not in dashboard response: {content}", content=dash)
            return False
        if dash["response"]["data"]["contact_count"] > 0:
            for contact in dash["response"]["data"]["contact_list"]:
                contact_id = contact["data"]["contact_id"]
                buyer = contact["data"]["buyer"]["username"]
                amount = contact["data"]["amount"]
                asset = contact["data"]["advertisement"]["asset"]
                if asset == "XMR":
                    amount_crypto = contact["data"]["amount_xmr"]
                elif asset == "BTC":
                    amount_crypto = contact["data"]["amount_btc"]
                currency = contact["data"]["currency"]
                release_url = contact["actions"]["release_url"]
                if not contact["data"]["is_selling"]:
                    continue
                reference = self.tx.tx_to_ref(contact_id)
                if not reference:
                    reference = "not_set"
                dash_tmp.append(f"{reference}: {buyer} {amount}{currency} {amount_crypto}{asset} {release_url}")

        return dash_tmp

    @handle_exceptions
    def get_recent_messages(self, send_irc=True):
        """
        Get recent messages.
        """
        messages_tmp = {}
        messages = self.agora.recent_messages()
        if messages is False:
            return False
        if not messages["success"]:
            return False
        if "data" not in messages["response"]:
            self.log.error("Data not in messages response: {content}", content=messages["response"])
            return False
        open_tx = self.tx.get_ref_map().keys()
        for message in messages["response"]["data"]["message_list"]:
            contact_id = message["contact_id"]
            username = message["sender"]["username"]
            msg = message["msg"]
            if contact_id not in open_tx:
                continue
            reference = self.tx.tx_to_ref(contact_id)
            if reference in messages_tmp:
                messages_tmp[reference].append([username, msg])
            else:
                messages_tmp[reference] = [[username, msg]]

            # Send new messages on IRC
            if send_irc:
                for user, message in messages_tmp[reference]:
                    if reference in self.last_messages:
                        if not [user, message] in self.last_messages[reference]:
                            self.irc.sendmsg(f"AUTO {reference}: ({user}) {message}")
                            # Append sent messages to last_messages so we don't send them again
                            self.last_messages[reference].append([user, message])
                    else:
                        self.last_messages[reference] = [[user, message]]
                        for x in messages_tmp[reference]:
                            self.irc.sendmsg(f"NEW {reference}: ({user}) {message}")

        # Purge old trades from cache
        for ref in list(self.last_messages):  # We're removing from the list on the fly
            if ref not in messages_tmp:
                del self.last_messages[ref]

        return messages_tmp

    @handle_exceptions
    def enum_ad_ids(self, page=0):
        ads = self.agora._api_call(api_method="ads", query_values={"page": page})
        if ads is False:
            return False
        ads_total = []
        if not ads["success"]:
            return False
        for ad in ads["response"]["data"]["ad_list"]:
            ads_total.append(ad["data"]["ad_id"])
        if "pagination" in ads["response"]:
            if "next" in ads["response"]["pagination"]:
                page += 1
                ads_iter = self.enum_ad_ids(page)
                if ads_iter is None:
                    return False
                if ads_iter is False:
                    return False
                for ad in ads_iter:
                    ads_total.append(ad)
        return ads_total

    @handle_exceptions
    def enum_ads(self, requested_asset=None, page=0):
        query_values = {"page": page}
        if requested_asset:
            query_values["asset"] = requested_asset
        ads = self.agora._api_call(api_method="ads", query_values=query_values)
        if ads is False:
            return False
        ads_total = []
        if not ads["success"]:
            return False
        for ad in ads["response"]["data"]["ad_list"]:
            asset = ad["data"]["asset"]
            ad_id = ad["data"]["ad_id"]
            country = ad["data"]["countrycode"]
            currency = ad["data"]["currency"]
            provider = ad["data"]["online_provider"]
            ads_total.append([asset, ad_id, country, currency, provider])
        if "pagination" in ads["response"]:
            if "next" in ads["response"]["pagination"]:
                page += 1
                ads_iter = self.enum_ads(requested_asset, page)
                if ads_iter is None:
                    return False
                if ads_iter is False:
                    return False
                for ad in ads_iter:
                    ads_total.append([ad[0], ad[1], ad[2], ad[3], ad[4]])
        return ads_total

    def last_online_recent(self, date):
        """
        Check if the last online date was recent.
        :param date: date last online
        :type date: string
        :return: bool indicating whether the date was recent enough
        :rtype: bool
        """
        date_parsed = datetime.strptime(date, "%Y-%m-%dT%H:%M:%S.%fZ")
        now = datetime.now()
        sec_ago_date = (now - date_parsed).total_seconds()
        self.log.debug("Seconds ago date for {date} ^ {now}: {x}", date=date, now=str(now), x=sec_ago_date)
        return sec_ago_date < 172800

    @handle_exceptions
    def enum_public_ads(self, coin, currency, providers=None, page=0):
        if not providers:
            providers = ["REVOLUT"]
        # buy-monero-online, buy-bitcoin-online
        # Work around Agora weirdness calling it bitcoins
        if coin == "bitcoin":
            coin = "bitcoins"
        if len(providers) == 1:
            ads = self.agora._api_call(api_method=f"buy-{coin}-online/{currency}/{providers[0]}", query_values={"page": page})
        elif len(providers) > 1:
            ads = self.agora._api_call(api_method=f"buy-{coin}-online/{currency}", query_values={"page": page})
        if ads is None:
            return False
        if ads is False:
            return False
        if "data" not in ads["response"]:
            return False
        for ad in ads["response"]["data"]["ad_list"]:
            if ad["data"]["online_provider"] not in providers:
                continue
            date_last_seen = ad["data"]["profile"]["last_online"]
            # Check if this person was seen recently
            if not self.last_online_recent(date_last_seen):
                continue
            ad_id = ad["data"]["ad_id"]
            username = ad["data"]["profile"]["username"]
            temp_price = ad["data"]["temp_price"]
            provider = ad["data"]["online_provider"]
            yield [ad_id, username, temp_price, provider]
        if "pagination" in ads["response"]:
            if "next" in ads["response"]["pagination"]:
                page += 1
                ads_iter = self.enum_public_ads(coin, currency, providers, page)
                if ads_iter is None:
                    return False
                if ads_iter is False:
                    return False
                for ad in ads_iter:
                    yield [ad[0], ad[1], ad[2], ad[3]]

    def wrap_public_ads(self, asset, currency, providers=None, rates=None):
        """
        Wrapper to sort public ads.
        """
        if asset == "XMR":
            coin = "monero"
        elif asset == "BTC":
            coin = "bitcoin"
        ads_obj = self.enum_public_ads(coin, currency.upper(), providers)
        ads = list(ads_obj)
        if ads is False:
            return False
        # ads = list(ads_obj)
        if ads is False:
            return False
        if not rates:
            # Set the price based on the asset
            base_currency_price = self.cg.get_price(ids=coin, vs_currencies=currency)[coin][currency.lower()]
        else:
            base_currency_price = rates
        for ad in ads:
            price = float(ad[2])
            rate = round(price / base_currency_price, 2)
            ad.append(rate)
        return sorted(ads, key=lambda x: x[2])

    def _update_prices(self, xmr=None, btc=None):
        """
        Update prices in another thread.
        """
        if xmr is None and btc is None:
            all_assets = loads(settings.Agora.AssetList)
            assets_not_run = set(all_assets) ^ set(self.cheat_run_on)
            if not assets_not_run:
                self.cheat_run_on = []
                asset = list(all_assets).pop()
                self.cheat_run_on.append(asset)
            else:
                asset = assets_not_run.pop()
                self.cheat_run_on.append(asset)
            if asset == "XMR":
                deferToThread(self.update_prices, True, False)  # XMR, BTC
            elif asset == "BTC":
                deferToThread(self.update_prices, False, True)  # XMR, BTC
            return asset
        else:
            deferToThread(self.update_prices, xmr, btc)

    @handle_exceptions
    def update_prices(self, xmr=True, btc=True):
        if xmr and btc:
            our_ads = self.enum_ads()
        elif xmr and not btc:
            our_ads = self.enum_ads("XMR")
        elif not xmr and btc:
            our_ads = self.enum_ads("BTC")

        self.log.debug("Our ads: {ads}", ads=our_ads)
        to_update = []
        if our_ads is None:
            return False
        if our_ads is False:
            return False
        currencies = [x[3].lower() for x in our_ads]
        # Sets deduplicate by default
        providers = list(set([x[4] for x in our_ads]))
        public_ad_dict_xmr = {}
        public_ad_dict_btc = {}
        rates_crypto = self.cg.get_price(ids=["monero", "bitcoin"], vs_currencies=currencies)

        # NOTES:
        # Get all ads for each currency, with all the payment methods.
        # Create a function to, in turn, filter these so it contains only one payment method. Run autoprice on this.
        # Append all results to to_update. Repeat for remaining payment methods, then call slow update.

        for currency in currencies:
            try:
                rates_xmr = rates_crypto["monero"][currency]
                rates_btc = rates_crypto["bitcoin"][currency]
                if xmr:
                    public_ad_dict_xmr[currency] = self.wrap_public_ads("XMR", currency, providers=providers, rates=rates_xmr)
                    if public_ad_dict_xmr[currency] is False:
                        return False
                if btc:
                    public_ad_dict_btc[currency] = self.wrap_public_ads("BTC", currency, providers=providers, rates=rates_btc)
                    if public_ad_dict_btc[currency] is False:
                        return False
            except KeyError:
                self.log.error("Error getting public ads for currency {currency}", currency=currency)
                continue
        for asset, ad_id, country, currency, provider in our_ads:
            # Get the ads for this country/currency pair
            try:
                if asset == "XMR":
                    public_ads = public_ad_dict_xmr[currency.lower()]
                elif asset == "BTC":
                    public_ads = public_ad_dict_btc[currency.lower()]
            except KeyError:
                continue
            if not public_ads:
                continue
            if xmr:
                if asset == "XMR":
                    new_margin = self.autoprice(public_ads, currency)
                    new_formula = f"coingeckoxmrusd*usd{currency.lower()}*{new_margin}"
            if btc:
                if asset == "BTC":
                    new_margin = self.autoprice(public_ads, currency)
                    new_formula = f"coingeckobtcusd*usd{currency.lower()}*{new_margin}"
            # Get all of our ads
            our_ads_list = [ad for ad in public_ads if ad[1] == settings.Agora.Username]
            if not len(our_ads_list) == 1:
                if not len(set([x[3] for x in our_ads_list])) == 1:
                    self.log.error("Our ads have different margins: {ads}", ads=our_ads_list)
            # Get one from the list, they're probably the same, if not we will deal with the other
            #  ones in a later pass

            # Get one
            our_ad = our_ads_list.pop()

            # Take the 4th argument, the margin
            our_margin = our_ad[3]

            # Don't waste API rate limits on setting the same margin as before
            if new_margin != our_margin:
                # rtrn = self.agora.ad_equation(ad_id, new_formula)
                to_update.append([ad_id, new_formula, asset, currency, False])
                # if not rtrn["success"]:
                #     self.log.error("Error updating ad {ad_id}: {response}", ad_id=ad_id, response=rtrn["response"])
                self.log.info("Rate for {currency}: {margin}", currency=currency, margin=new_margin)
            else:
                self.log.info("Not changed rate for {currency}, keeping old margin of {margin}", currency=currency, margin=our_margin)
        self.slow_ad_update(to_update)

    def slow_ad_update(self, ads):
        """
        Slow ad equation update utilising exponential backoff in order to guarantee all ads are updated.
        :param ads: our list of ads
        """
        self.log.info("Beginning slow ad update for {num} ads", num=len(ads))
        iterations = 0
        throttled = 0
        assets = set()
        currencies = set()
        while not all([x[4] for x in ads]) or iterations == 1000:
            for ad_index in range(len(ads)):
                ad_id, new_formula, asset, currency, actioned = ads[ad_index]
                assets.add(asset)
                currencies.add(currency)
                self.log.error("ASSET {a}", a=asset)
                if not actioned:
                    rtrn = self.agora.ad_equation(ad_id, new_formula)
                    if rtrn["success"]:
                        ads[ad_index][4] = True
                        throttled = 0
                        self.log.info("Successfully updated ad: {id}", id=ad_id)
                        continue
                    else:
                        if rtrn["response"]["error"]["error_code"] == 429:
                            throttled += 1
                            sleep_time = pow(throttled, float(settings.Agora.SleepExponent))
                            self.log.info(
                                "Throttled {x} times while updating {id}, sleeping for {sleep} seconds",
                                x=throttled,
                                id=ad_id,
                                sleep=sleep_time,
                            )
                            # We're running in a thread, so this is fine
                            sleep(sleep_time)
                        self.log.error("Error updating ad {ad_id}: {response}", ad_id=ad_id, response=rtrn["response"])
                        continue
            iterations += 1
        if iterations == 0:
            self.log.info("Slow ad update finished, no ads to update")
            self.irc.sendmsg("Slow ad update finished, no ads to update")
        else:
            self.log.info(
                "Slow ad update completed with {iterations} iterations: [{assets}] | [{currencies}]",
                iterations=iterations,
                assets=", ".join(assets),
                currencies=", ".join(currencies),
            )
            self.irc.sendmsg(f"Slow ad update completed with {iterations} iterations: [{', '.join(assets)}] | [{', '.join(currencies)}]")

    def autoprice(self, ads, currency):
        """
        Helper function to automatically adjust the price up/down in certain markets
        in order to gain the most profits and sales.
        :param ads: list of ads
        :type ads: list of lists
        :param currency: currency of the ads
        :type currency: string
        :return: the rate we should use for this currency
        :rtype: float
        """
        self.log.debug("Autoprice starting for {x}", x=currency)
        # Find cheapest ad
        # Filter by 3rd index on each ad list to find the cheapest
        min_margin_ad = min(ads, key=lambda x: x[4])
        self.log.debug("Minimum margin ad: {x}", x=min_margin_ad)

        # Find second cheapest that is not us
        # Remove results from ads that are us
        ads_without_us = [ad for ad in ads if not ad[1] == settings.Agora.Username]
        self.log.debug("Ads without us: {x}", x=ads_without_us)
        # Find ads above our min that are not us
        ads_above_our_min_not_us = [ad for ad in ads_without_us if ad[4] > float(settings.Agora.MinMargin)]
        self.log.debug("Ads above our min not us: {x}", x=ads_above_our_min_not_us)
        # Check that this list without us is not empty
        if ads_without_us:
            # Find the cheapest from these
            min_margin_ad_not_us = min(ads_without_us, key=lambda x: x[4])
            self.log.debug("Min margin ad not us: {x}", x=min_margin_ad_not_us)
            # Lowball the lowest ad that is not ours
            lowball_lowest_not_ours = min_margin_ad_not_us[4]  # - 0.005
            self.log.debug("Lowball lowest not ours: {x}", x=lowball_lowest_not_ours)

        # Check if the username field of the cheapest ad matches ours
        if min_margin_ad[1] == settings.Agora.Username:
            self.log.debug("We are the cheapest for: {x}", x=currency)
            # We are the cheapest!
            # Are all of the ads ours?
            all_ads_ours = all([ad[1] == settings.Agora.Username for ad in ads])
            if all_ads_ours:
                self.log.debug("All ads are ours for: {x}", x=currency)
                # Now we know it's safe to return the maximum value
                return float(settings.Agora.MaxMargin)
            else:
                self.log.debug("All ads are NOT ours for: {x}", x=currency)
                # All the ads are not ours, but we are first...
                # Check if the lowballed, lowest (that is not ours) ad's margin
                # is less than our minimum
                if lowball_lowest_not_ours < float(settings.Agora.MinMargin):
                    self.log.debug("Lowball lowest not ours less than MinMargin")
                    return float(settings.Agora.MinMargin)
                elif lowball_lowest_not_ours > float(settings.Agora.MaxMargin):
                    self.log.debug("Lowball lowest not ours more than MaxMargin")
                    return float(settings.Agora.MaxMargin)
                else:
                    self.log.debug("Returning lowballed figure: {x}", x=lowball_lowest_not_ours)
                    return lowball_lowest_not_ours
        else:
            self.log.debug("We are NOT the cheapest for: {x}", x=currency)
            # We are not the cheapest :(
            # Check if this list is empty
            if not ads_above_our_min_not_us:
                # Return the maximum margin?
                return float(settings.Agora.MaxMargin)
            # Find cheapest ad above our min that is not us
            cheapest_ad = min(ads_above_our_min_not_us, key=lambda x: x[4])
            cheapest_ad_margin = cheapest_ad[4]  # - 0.005
            if cheapest_ad_margin > float(settings.Agora.MaxMargin):
                self.log.debug("Cheapest ad not ours more than MaxMargin")
                return float(settings.Agora.MaxMargin)
            self.log.debug("Cheapest ad above our min that is not us: {x}", x=cheapest_ad)
            return cheapest_ad_margin

    @handle_exceptions
    def nuke_ads(self):
        """
        Delete all of our adverts.
        :return: True or False
        :rtype: bool
        """
        ads = self.enum_ad_ids()
        return_ids = []
        if ads is False:
            return False
        for ad_id in ads:
            rtrn = self.agora.ad_delete(ad_id)
            return_ids.append(rtrn["success"])
        return all(return_ids)

    def get_rates_all(self):
        """
        Get all rates that pair with USD.
        :return: dictionary of USD/XXX rates
        :rtype: dict
        """
        rates = self.cr.get_rates("USD")
        return rates

    def get_acceptable_margins(self, currency, amount):
        """
        Get the minimum and maximum amounts we would accept a trade for.
        :param currency: currency code
        :param amount: amount
        :return: (min, max)
        :rtype: tuple
        """
        rates = self.get_rates_all()
        if currency == "USD":
            min_amount = amount - float(settings.Agora.AcceptableUSDMargin)
            max_amount = amount + float(settings.Agora.AcceptableUSDMargin)
            return (min_amount, max_amount)
        amount_usd = amount / rates[currency]
        min_usd = amount_usd - float(settings.Agora.AcceptableUSDMargin)
        max_usd = amount_usd + float(settings.Agora.AcceptableUSDMargin)
        min_local = min_usd * rates[currency]
        max_local = max_usd * rates[currency]
        return (min_local, max_local)

    @handle_exceptions
    def create_ad(self, asset, countrycode, currency, provider, edit=False, ad_id=None):
        """
        Post an ad with the given asset in a country with a given currency.
        Convert the min and max amounts from settings to the given currency with CurrencyRates.
        :param asset: the crypto asset to list (XMR or BTC)
        :type asset: string
        :param countrycode: country code
        :param currency: currency code
        :type countrycode: string
        :type currency: string
        :return: data about created object or error
        :rtype: dict
        """
        ad = settings.Agora.Ad
        paymentdetails = settings.Agora.PaymentDetails

        # Substitute the currency
        ad = ad.replace("$CURRENCY$", currency)
        if countrycode == "GB" and currency == "GBP":
            ad = ad.replace("$PAYMENT$", settings.Agora.GBPDetailsAd)
            paymentdetailstext = paymentdetails.replace("$PAYMENT$", settings.Agora.GBPDetailsPayment)
        else:
            ad = ad.replace("$PAYMENT$", settings.Agora.DefaultDetailsAd)
            paymentdetailstext = paymentdetails.replace("$PAYMENT$", settings.Agora.DefaultDetailsPayment)

        # Substitute the asset
        ad = ad.replace("$ASSET$", asset)

        rates = self.get_rates_all()
        if asset == "XMR":
            min_usd = float(settings.Agora.MinUSDXMR)
            max_usd = float(settings.Agora.MaxUSDXMR)
        elif asset == "BTC":
            min_usd = float(settings.Agora.MinUSDBTC)
            max_usd = float(settings.Agora.MaxUSDBTC)
        if currency == "USD":
            min_amount = min_usd
            max_amount = max_usd
        else:
            min_amount = rates[currency] * min_usd
            max_amount = rates[currency] * max_usd
        price_formula = f"coingecko{asset.lower()}usd*usd{currency.lower()}*{settings.Agora.Margin}"
        # price_formula = f"coingeckoxmrusd*{settings.Agora.Margin}"

        # Remove extra tabs
        ad = ad.replace("\\t", "\t")

        form = {
            "country_code": countrycode,
            "currency": currency,
            "trade_type": "ONLINE_SELL",
            "asset": asset,
            "price_equation": price_formula,
            "track_max_amount": False,
            "require_trusted_by_advertiser": False,
            "online_provider": provider,
            "msg": ad,
            "min_amount": min_amount,
            "max_amount": max_amount,
            "payment_method_details": settings.Agora.PaymentMethodDetails,
            "account_info": paymentdetailstext,
        }

        # Dirty hack to test
        # if asset == "BTC":
        #    del form["min_amount"]
        if edit:
            ad = self.agora.ad(ad_id=ad_id, **form)
        else:
            ad = self.agora.ad_create(**form)
        return ad

    def create_distribution_list(self):
        """
        Create a list for distribution of ads.
        :return: generator of asset, countrycode, currency, provider
        :rtype: generator of tuples
        """
        # Iterate providers like REVOLUT, NATIONAL_BANK
        for provider in loads(settings.Agora.ProviderList):
            # Iterate assets like XMR, BTC
            for asset in loads(settings.Agora.AssetList):
                # Iterate pairs of currency and country like EUR, GB
                for currency, countrycode in loads(settings.Agora.DistList):
                    yield (asset, countrycode, currency, provider)

    def dist_countries(self):
        """
        Distribute our advert into all countries and providers listed in the config.
        Exits on errors.
        :return: False or dict with response
        :rtype: bool or dict
        """
        dist_list = list(self.create_distribution_list())
        our_ads = self.enum_ads()
        # Let's get rid of the ad IDs and make it a tuple like dist_list
        our_ads = [(x[0], x[2], x[3], x[4]) for x in our_ads]
        for asset, countrycode, currency, provider in dist_list:
            if (asset, countrycode, currency, provider) not in our_ads:
                # Create the actual ad and pass in all the stuff
                rtrn = self.create_ad(asset, countrycode, currency, provider)
                # Bail on first error, let's not continue
                if rtrn is False:
                    return False
                yield rtrn

    def redist_countries(self):
        """
        Redistribute our advert details into all our listed adverts.
        This will edit all ads and update the details. Only works if we have already run dist.
        This will not post any new ads.
        Exits on errors.
        :return: False or dict with response
        :rtype: bool or dict
        """
        our_ads = self.enum_ads()
        for asset, ad_id, countrycode, currency, provider in our_ads:
            rtrn = self.create_ad(asset, countrycode, currency, provider, edit=True, ad_id=ad_id)
            # Bail on first error, let's not continue
            if rtrn is False:
                return False
            yield (rtrn, ad_id)

    @handle_exceptions
    def strip_duplicate_ads(self):
        """
        Remove duplicate ads.
        :return: list of duplicate ads
        :rtype: list
        """
        existing_ads = self.enum_ads()
        _size = len(existing_ads)
        repeated = []
        for i in range(_size):
            k = i + 1
            for j in range(k, _size):
                if existing_ads[i] == existing_ads[j] and existing_ads[i] not in repeated:
                    repeated.append(existing_ads[i])

        actioned = []
        for ad_id, country, currency in repeated:
            rtrn = self.agora.ad_delete(ad_id)
            actioned.append(rtrn["success"])
        return all(actioned)

    @handle_exceptions
    def release_funds(self, contact_id):
        """
        Release funds for a contact_id.
        :param contact_id: trade/contact ID
        :type contact_id: string
        :return: response dict
        :rtype: dict
        """
        payload = {"tradeId": contact_id, "password": settings.Agora.Pass}
        rtrn = self.agora._api_call(api_method=f"contact_release/{contact_id}", http_method="POST", query_values=payload)

        # Check if we can withdraw funds
        self.withdraw_funds()

        return rtrn

    @handle_exceptions
    def withdraw_funds(self):
        """
        Withdraw excess funds to our XMR/BTC wallets.
        """
        totals_all = self.tx.get_total()
        if totals_all is False:
            return False

        wallet_xmr, _ = totals_all[2]

        # Get the wallet balances in USD
        total_usd = totals_all[0][1]

        profit_usd = total_usd - float(settings.Money.BaseUSD)

        # Get the XMR -> USD exchange rate
        xmr_usd = self.cg.get_price(ids="monero", vs_currencies=["USD"])

        # Convert the USD total to XMR
        profit_usd_in_xmr = float(profit_usd) / xmr_usd["monero"]["usd"]

        # Check profit is above zero
        if not profit_usd >= 0:
            return

        if not float(wallet_xmr) > profit_usd_in_xmr:
            # Not enough funds to withdraw
            self.log.error(
                "Not enough funds to withdraw {profit}, as wallet only contains {wallet}", profit=profit_usd_in_xmr, wallet=wallet_xmr
            )
            self.irc.sendmsg(f"Not enough funds to withdraw {profit_usd_in_xmr}, as wallet only contains {wallet_xmr}")
            return

        if not profit_usd >= float(settings.Money.WithdrawLimit):
            # Not enough profit to withdraw
            return

        half = profit_usd_in_xmr / 2

        half_rounded = round(half, 8)

        # Read OTP secret
        with open("otp.key", "r") as f:
            otp_key = f.read()
        f.close()
        otp_key = otp_key.replace("\n", "")

        # Get OTP code
        otp_code = TOTP(otp_key)

        # Set up the format for calling wallet_send_xmr
        send_cast = {
            "address": None,
            "amount": half_rounded,
            "password": settings.Agora.Pass,
            "otp": otp_code.now(),
        }

        send_cast["address"] = settings.XMR.Wallet1
        rtrn1 = self.agora.wallet_send_xmr(**send_cast)

        send_cast["address"] = settings.XMR.Wallet2
        rtrn2 = self.agora.wallet_send_xmr(**send_cast)

        self.irc.sendmsg(f"Withdrawal: {rtrn1['success']} | {rtrn2['success']}")
        self.notify.notify_withdrawal(profit_usd / 2)

    def to_usd(self, amount, currency):
        if currency == "USD":
            return float(amount)
        else:
            rates = self.get_rates_all()
            return float(amount) / rates[currency]