Implement propagating account data to ads

handler/agora.py

@@ -454,36 +454,49 @@ class Agora(util.Base):
             return_ids.append(rtrn["success"])
         return all(return_ids)
 
-    @util.handle_exceptions
+    def format_ad(self, asset, currency, payment_details_text):
-    def create_ad(self, asset, countrycode, currency, provider, edit=False, ad_id=None):
         """
-        Post an ad with the given asset in a country with a given currency.
+        Format the ad.
-        Convert the min and max amounts from settings to the given currency with CurrencyRates.
-        :param asset: the crypto asset to list (XMR or BTC)
-        :type asset: string
-        :param countrycode: country code
-        :param currency: currency code
-        :type countrycode: string
-        :type currency: string
-        :return: data about created object or error
-        :rtype: dict
         """
         ad = settings.Agora.Ad
-        paymentdetails = settings.Agora.PaymentDetails
 
         # Substitute the currency
         ad = ad.replace("$CURRENCY$", currency)
-        if currency == "GBP":
-            ad = ad.replace("$PAYMENT$", settings.Agora.GBPDetailsAd)
-            paymentdetailstext = paymentdetails.replace("$PAYMENT$", settings.Agora.GBPDetailsPayment)
-        else:
-            ad = ad.replace("$PAYMENT$", settings.Agora.DefaultDetailsAd)
-            paymentdetailstext = paymentdetails.replace("$PAYMENT$", settings.Agora.DefaultDetailsPayment)
 
         # Substitute the asset
         ad = ad.replace("$ASSET$", asset)
 
+        # Substitute the payment details
+        ad = ad.replace("$PAYMENT$", payment_details_text)
+
+        # Strip extra tabs
+        ad = ad.replace("\\t", "\t")
+        return ad
+
+    def format_payment_details(self, currency, payment_details):
+        """
+        Format the payment details.
+        """
+        payment = settings.Agora.PaymentDetails
+
+        payment_text = ""
+        for field, value in payment_details.items():
+            formatted_name = field.replace("_", " ")
+            formatted_name = formatted_name.capitalize()
+            payment_text += f"* {formatted_name}: **{value}**"
+            if field != list(payment_details.keys())[-1]:  # No trailing newline
+                payment_text += "\n"
+
+        payment = payment.replace("$PAYMENT$", payment_text)
+        payment = payment.replace("$CURRENCY$", currency)
+
+        return payment
+
+    def get_minmax(self, asset, currency):
         rates = self.money.get_rates_all()
+        if currency not in rates and not currency == "USD":
+            self.log.error(f"Can't create ad without rates: {currency}")
+            return
         if asset == "XMR":
             min_usd = float(settings.Agora.MinUSDXMR)
             max_usd = float(settings.Agora.MaxUSDXMR)
@@ -496,9 +509,33 @@ class Agora(util.Base):
         else:
             min_amount = rates[currency] * min_usd
             max_amount = rates[currency] * max_usd
+
+        return (min_amount, max_amount)
+
+    @util.handle_exceptions
+    def create_ad(self, asset, countrycode, currency, provider, payment_details, visible=True, edit=False, ad_id=None):
+        """
+        Post an ad with the given asset in a country with a given currency.
+        Convert the min and max amounts from settings to the given currency with CurrencyRates.
+        :param asset: the crypto asset to list (XMR or BTC)
+        :type asset: string
+        :param countrycode: country code
+        :param currency: currency code
+        :param payment_details: the payment details
+        :type countrycode: string
+        :type currency: string
+        :type payment_details: dict
+        :return: data about created object or error
+        :rtype: dict
+        """
+
+        if payment_details:
+            payment_details_text = self.format_payment_details(currency, payment_details)
+            ad_text = self.format_ad(asset, currency, payment_details_text)
+            min_amount, max_amount = self.get_minmax(asset, currency)
+
         price_formula = f"coingecko{asset.lower()}usd*usd{currency.lower()}*{settings.Agora.Margin}"
-        # Remove extra tabs
+
-        ad = ad.replace("\\t", "\t")
         form = {
             "country_code": countrycode,
             "currency": currency,
@@ -508,12 +545,15 @@ class Agora(util.Base):
             "track_max_amount": False,
             "require_trusted_by_advertiser": False,
             "online_provider": provider,
-            "msg": ad,
-            "min_amount": min_amount,
-            "max_amount": max_amount,
             "payment_method_details": settings.Agora.PaymentMethodDetails,
-            "account_info": paymentdetailstext,
+            "visible": visible,
         }
+        if payment_details:
+            form["account_info"] = payment_details_text
+            form["msg"] = ad_text
+            form["min_amount"] = min_amount
+            form["max_amount"] = max_amount
+
         if edit:
             ad = self.agora.ad(ad_id=ad_id, **form)
         else:

handler/agoradesk_py.py

@@ -505,7 +505,7 @@ class AgoraDesk:
         if lon:
             params["lon"] = lon
         if visible:
-            params["visible"] = 1 if visible else 0
+            params["visible"] = True if visible else False
 
         return self._api_call(
             api_method=f"ad/{ad_id}",

handler/markets.py

@@ -165,3 +165,48 @@ class Markets(util.Base):
                     if filter_asset:
                         if asset == filter_asset:
                             yield (asset, countrycode, currency, provider)
+
+    def distribute_account_details(self, currencies=None, account_info=None):
+        """
+        Distribute account details for ads.
+        We will disable ads we can't support.
+        """
+        if not currencies:
+            currencies = self.sinks.currencies
+        if not account_info:
+            account_info = self.sinks.account_info
+        # First, let's get the ads we can't support
+        all_currencies = self.get_all_currencies()
+        supported_currencies = [currency for currency in currencies if currency in all_currencies]
+
+        # not_supported = [currency for currency in all_currencies if currency not in supported_currencies]
+
+        our_ads = self.agora.enum_ads()
+
+        supported_ads = [ad for ad in our_ads if ad[3] in supported_currencies]
+
+        not_supported_ads = [ad for ad in our_ads if ad[3] not in supported_currencies]
+
+        currency_account_info_map = {}
+        for currency in supported_currencies:
+            for bank, accounts in account_info.items():
+                for account in accounts:
+                    if account["currency"] == currency:
+                        currency_account_info_map[currency] = account["account_number"]
+
+        for ad in supported_ads:
+            asset = ad[0]
+            countrycode = ad[2]
+            currency = ad[3]
+            provider = ad[4]
+            payment_details = currency_account_info_map[currency]
+            ad_id = ad[1]
+            self.agora.create_ad(asset, countrycode, currency, provider, payment_details, visible=True, edit=True, ad_id=ad_id)
+
+        for ad in not_supported_ads:
+            asset = ad[0]
+            countrycode = ad[2]
+            currency = ad[3]
+            provider = ad[4]
+            ad_id = ad[1]
+            self.agora.create_ad(asset, countrycode, currency, provider, payment_details=False, visible=False, edit=True, ad_id=ad_id)

handler/sinks/__init__.py

@@ -18,23 +18,61 @@ class Sinks(util.Base):
 
     def __init__(self):
         super().__init__()
+        self.account_info = {}
+
+    def __irc_started__(self):
+        self.startup()
+
+    def startup(self):
+        """
+        We NEED the other libraries, and we initialise fast, so don't make
+        any race conditions by relying on something that might not be there.
+        """
         self.fidor = sinks.fidor.Fidor()
         self.nordigen = sinks.nordigen.Nordigen()
         self.truelayer = sinks.truelayer.TrueLayer(self)
         # setattr(self.truelayer, "sinks", self)
 
     def got_transactions(self, bank, account_id, transactions):
-        print("GOT transactions", bank, account_id, transactions)
+        if not transactions:
+            return False
         transaction_ids = [x["transaction_id"] for x in transactions]
-        print("IDS", transaction_ids)
         new_key_name = f"new.transactions.{bank}.{account_id}"
         old_key_name = f"transactions.{bank}.{account_id}"
-        r.sset(new_key_name, transaction_ids)
+        # for transaction_id in transaction_ids:
+        if not transaction_ids:
+            return
+        r.sadd(new_key_name, *transaction_ids)
+
+        difference = list(r.sdiff(new_key_name, old_key_name))
+
+        difference = util.convert(difference)
 
-        difference = r.sdiff(new_key_name, old_key_name)
+        new_transactions = [x for x in transactions if x["transaction_id"] in difference]
-        print("difference", difference)
 
         # Rename the new key to the old key so we can run the diff again
         r.rename(new_key_name, old_key_name)
+        for transaction in new_transactions:
+            self.tx.transaction(transaction)
+
+    def got_account_info(self, subclass, account_infos):
+        """
+        Called when we get account information from an API provider.
+        :param subclass: class name that called it, truelayer, fidor, etc
+        :param account_infos: dict of dicts of account information
+        :param account_infos: dict
+        """
+
+        for bank, accounts in account_infos.items():
+            for account in list(accounts):
+                if len(account["account_number"]) == 1:
+                    account_infos[bank].remove(account)
+        currencies = [account["currency"] for bank, accounts in account_infos.items() for account in accounts]
+
+        self.account_info = account_infos
+        self.currencies = currencies
 
-        # self.transactions.transaction(transactions)
+        # parsed_details =
+        # {"EUR": {"IBAN": "xxx", "BIC": "xxx"},
+        #  "GBP": {"SORT": "04-04-04", "ACCOUNT": "1922-2993"}}
+        # self.markets.distribute_account_details(currencies, account_infos)

handler/sinks/truelayer.py

@@ -23,6 +23,8 @@ class TrueLayer(util.Base):
         self.sinks = sinks
         self.tokens = {}
         self.banks = {}
+        self.refresh_tokens = {}
+        self.authed = False
 
         # Get the banks from the config and cache them
         self.get_mapped_accounts()
@@ -35,6 +37,15 @@ class TrueLayer(util.Base):
         # -> set self.tokens[bank] = access_token
         self.lc.start(int(settings.TrueLayer.TokenRefreshSec))
 
+    def __authed__(self):
+        """
+        Called when we have received all the API tokens.
+        """
+        # Get the account information and pass it to the main function
+        self.log.info("All accounts authenticated: " + ", ".join(self.tokens.keys()))
+        account_infos = self.get_all_account_info()
+        self.sinks.got_account_info("truelayer", account_infos)
+
         self.lc_tx = LoopingCall(self.transaction_loop)
         self.lc_tx.start(int(settings.TrueLayer.RefreshSec))
 
@@ -123,8 +134,12 @@ class TrueLayer(util.Base):
         refresh_tokens = loads(settings.TrueLayer.RefreshKeys)
         # Set the cached entry
         self.refresh_tokens = refresh_tokens
+
         for bank in refresh_tokens:
-            self.get_new_token(bank)
+            rtrn = self.get_new_token(bank)
+            if not rtrn:
+                self.log.error(f"Error getting token for {bank}")
+                return
 
     def get_new_token(self, bank):
         """
@@ -133,6 +148,7 @@ class TrueLayer(util.Base):
         :type account:
         """
         if bank not in self.refresh_tokens:
+            self.log.error(f"Bank {bank} not in refresh tokens")
             return
 
         headers = {"Content-Type": "application/x-www-form-urlencoded"}
@@ -146,17 +162,22 @@ class TrueLayer(util.Base):
         try:
             parsed = r.json()
         except JSONDecodeError:
+            self.log.error(f"Failed to decode JSON: {r.content}")
             return False
         if r.status_code == 200:
             if "access_token" in parsed.keys():
                 self.tokens[bank] = parsed["access_token"]
-                self.log.info(f"Refreshed access token for {bank}")
+                # self.log.info(f"Refreshed access token for {bank}")
+                if len(self.refresh_tokens.keys()) == len(self.tokens.keys()) and not self.authed:
+                    # We are now fully authenticated and ready to start loops!
+                    self.__authed__()
+                    self.authed = True
                 return True
             else:
-                self.log.error(f"Token refresh didn't contain access token: {parsed}", parsed=parsed)
+                self.log.error(f"Token refresh didn't contain access token: {parsed}")
                 return False
         else:
-            self.log.error(f"Cannot refresh token: {parsed}", parsed=parsed)
+            self.log.error(f"Cannot refresh token: {parsed}")
             return False
 
     def get_accounts(self, bank):
@@ -183,7 +204,7 @@ class TrueLayer(util.Base):
         try:
             parsed = r.json()
         except JSONDecodeError:
-            self.log.error("Error parsing accounts response: {content}", content=r.content)
+            self.log.error(f"Error parsing accounts response: {r.content}")
             return False
 
         return parsed
@@ -192,6 +213,17 @@ class TrueLayer(util.Base):
         existing_entry = loads(settings.TrueLayer.Maps)
         self.banks = existing_entry
 
+    def get_all_account_info(self):
+        to_return = {}
+        for bank in self.banks:
+            for account_id in self.banks[bank]:
+                account_data = self.get_account(bank, account_id)
+                if bank in to_return:
+                    to_return[bank].append(account_data)
+                else:
+                    to_return[bank] = [account_data]
+        return to_return
+
     def get_account(self, bank, account_id):
         account_data = self._get_account(bank, account_id)
         if "results" not in account_data:
@@ -241,7 +273,9 @@ class TrueLayer(util.Base):
         try:
             parsed = r.json()
         except JSONDecodeError:
-            self.log.error("Error parsing transactions response: {content}", content=r.content)
+            self.log.error(f"Error parsing transactions response: {r.content}")
+            return False
+        if "results" in parsed:
+            return parsed["results"]
+        else:
             return False
-
-        return parsed["results"]

handler/tests/test_agora.py

@@ -229,3 +229,32 @@ class TestAgora(TestCase):
         # Test specifying rates=
         lookup_rates_return = self.agora.money.lookup_rates(enum_ads_return, rates=cg_prices)
         self.assertCountEqual(lookup_rates_return, expected_return)
+
+    def test_format_ad(self):
+        settings.settings.Agora.Ad = """* Set **Country of recipient's bank** to **"United Kingdom"**
+$PAYMENT$
+* Set **Company name** to **"PATHOGEN LIMITED"**"""
+        payment_details = {"sort_code": "02-03-04", "account_number": "0023-0045"}
+        payment_details_text = self.agora.format_payment_details("GBP", payment_details)
+        ad_text = self.agora.format_ad("XMR", "GBP", payment_details_text)
+        expected = """* Set **Country of recipient's bank** to **"United Kingdom"**
+* Company name: **PATHOGEN LIMITED**
+* Sort code: **02-03-04**
+* Account number: **0023-0045**
+* Please send in **GBP**
+* If you are asked for address information, please use **24 Holborn Viaduct, London, England, EC1A 2BN**
+* The post code is **EC1A 2BN**
+* Set **Company name** to **"PATHOGEN LIMITED"**"""
+        self.assertEqual(ad_text, expected)
+
+    def test_format_payment_details(self):
+        payment_details = {"sort_code": "02-03-04", "account_number": "0023-0045"}
+        payment_details_text = self.agora.format_payment_details("GBP", payment_details)
+
+        expected = """* Company name: **PATHOGEN LIMITED**
+* Sort code: **02-03-04**
+* Account number: **0023-0045**
+* Please send in **GBP**
+* If you are asked for address information, please use **24 Holborn Viaduct, London, England, EC1A 2BN**
+* The post code is **EC1A 2BN**"""
+        self.assertEqual(payment_details_text, expected)

handler/transactions.py

@@ -75,92 +75,29 @@ class Transactions(util.Base):
         :param data: details of transaction
         :type data: dict
         """
-        event = data["event"]
         ts = data["timestamp"]
-
+        txid = data["transaction_id"]
-        if "data" not in data:
+        txtype = data["transaction_type"]
-            return
+        amount = data["amount"]
-        inside = data["data"]
-
-        txid = inside["id"]
-
-        if "type" not in inside:
-            # stored_trade here is actually TX
-            stored_trade = r.hgetall(f"tx.{txid}")
-            if not stored_trade:
-                self.log.error(f"Could not find entry in DB for typeless transaction: {txid}")
-                return
-            stored_trade = util.convert(stored_trade)
-            if "old_state" in inside:
-                if "new_state" in inside:
-                    # We don't care unless we're being told a transaction is now completed
-                    if not inside["new_state"] == "completed":
-                        return
-                    # We don't care unless the existing trade is pending
-                    if not stored_trade["state"] == "pending":
-                        return
-                # Check the old state is what we also think it is
-                if inside["old_state"] == stored_trade["state"]:
-                    # Set the state to the new state
-                    stored_trade["state"] = inside["new_state"]
-                    # Store the updated state
-                    r.hmset(f"tx.{txid}", stored_trade)
-                    # Check it's all been previously validated
-                    if "valid" not in stored_trade:
-                        self.log.error(f"Valid not in stored trade for {txid}, aborting.")
-                        return
-                    if stored_trade["valid"] == "1":
-                        # Make it invalid immediately, as we're going to release now
-                        stored_trade["valid"] = "0"
-                        r.hmset(f"tx.{txid}", stored_trade)
-                        reference = self.tx_to_ref(stored_trade["trade_id"])
-                        self.release_funds(stored_trade["trade_id"], reference)
-                        self.ux.notify.notify_complete_trade(stored_trade["amount"], stored_trade["currency"])
-                        return
-            # If type not in inside and we haven't hit any more returns
-            return
-        else:
-            txtype = inside["type"]
-            if txtype == "card_payment":
-                self.log.info(f"Ignoring card payment: {txid}")
-                return
-
-        state = inside["state"]
-        if "reference" in inside:
-            reference = inside["reference"]
-        else:
-            reference = "not_given"
-
-        leg = inside["legs"][0]
-
-        if "counterparty" in leg:
-            account_type = leg["counterparty"]["account_type"]
-        else:
-            account_type = "not_given"
-
-        amount = leg["amount"]
         if amount <= 0:
             self.log.info(f"Ignoring transaction with negative/zero amount: {txid}")
             return
-        currency = leg["currency"]
+        currency = data["currency"]
-        description = leg["description"]
+        description = data["description"]
+        reference = data["meta"]["provider_reference"]
 
         to_store = {
-            "event": event,
             "trade_id": "",
             "ts": ts,
             "txid": txid,
             "txtype": txtype,
-            "state": state,
             "reference": reference,
-            "account_type": account_type,
             "amount": amount,
             "currency": currency,
             "description": description,
-            "valid": 0,  # All checks passed and we can release escrow?
         }
         self.log.info(f"Transaction processed: {dumps(to_store, indent=2)}")
-        self.irc.sendmsg(f"AUTO Incoming transaction: {amount}{currency} ({reference}) - {state} - {description}")
+        self.irc.sendmsg(f"AUTO Incoming transaction: {amount}{currency} ({reference}) - {description}")
         # Partial reference implementation
         # Account for silly people not removing the default string
         # Split the reference into parts
@@ -234,19 +171,9 @@ class Transactions(util.Base):
                 self.irc.sendmsg(f"Amount mismatch - not in margins: {stored_trade['amount']} (min: {min_amount} / max: {max_amount}")
                 return
 
-        # We have made it this far without hitting any of the returns, so let's set valid = True
-        # This will let us instantly release if the type is pending, and it is subsequently updated to completed with a callback.
-        to_store["valid"] = 1
-        # Store the trade ID so we can release it easily
-        to_store["trade_id"] = stored_trade["id"]
-        if not state == "completed":
-            self.log.info(f"Storing incomplete trade: {txid}")
-            r.hmset(f"tx.{txid}", to_store)
-            # Don't procees further if state is not "completed"
-            return
-
         r.hmset(f"tx.{txid}", to_store)
-        self.release_funds(stored_trade["id"], stored_trade["reference"])
+        # self.release_funds(stored_trade["id"], stored_trade["reference"])
+        print("WOULD RELEASE THE FUCKING MONEY")
         self.ux.notify.notify_complete_trade(amount, currency)
 
     def release_funds(self, trade_id, reference):

handler/ux/commands.py

@@ -477,7 +477,6 @@ class IRCCommands(object):
                 account_id = spl[2]
                 transactions = tx.sinks.truelayer.get_transactions(account, account_id)
                 for transaction in transactions:
-                    print(transaction)
                     txid = transaction["transaction_id"]
                     ptxid = transaction["provider_transaction_id"]
                     txtype = transaction["transaction_type"]
@@ -502,3 +501,31 @@ class IRCCommands(object):
                     msg(f"Failed to map the account")
                     return
                 msg(f"Mapped account ID {account_id} at bank {bank} to {account_name}")
+
+    class unmapped(object):
+        name = "unmapped"
+        authed = True
+        helptext = "Get unmapped accounts for a bank. Usage: unmapped <bank>"
+
+        @staticmethod
+        def run(cmd, spl, length, authed, msg, agora, tx, ux):
+            if length == 2:
+                bank = spl[1]
+                accounts_active = []
+                for bank, accounts in tx.sinks.truelayer.banks.items():
+                    for account in accounts:
+                        accounts_active.append(account)
+                accounts_all = tx.sinks.truelayer.get_accounts(bank)
+                accounts_unmapped = [x["account_id"] for x in accounts_all["results"] if x["account_id"] not in accounts_active]
+                msg(f"Unmapped accounts: {', '.join(accounts_unmapped)}")
+
+    class distdetails(object):
+        name = "distdetails"
+        authed = True
+        helptext = "Distribute account details among all ads."
+
+        @staticmethod
+        def run(cmd, spl, length, authed, msg, agora, tx, ux):
+            currencies = tx.sinks.currencies
+            tx.markets.distribute_account_details()
+            msg(f"Distributing account details for currencies: {', '.join(currencies)}")

handler/ux/irc.py

@@ -118,6 +118,7 @@ class IRCBot(irc.IRCClient):
         :type channel: string
         """
         self.agora.setup_loop()
+        self.sinks.__irc_started__()
         self.log.info(f"Joined channel {channel}")
 
     def privmsg(self, user, channel, msg):