Implement propagating account data to ads

This commit is contained in:
Mark Veidemanis 2022-03-08 20:42:47 +00:00
parent aefa6c58a4
commit f2c9725fcb
Signed by: m
GPG Key ID: 5ACFCEED46C0904F
9 changed files with 264 additions and 123 deletions

View File

@ -454,36 +454,49 @@ class Agora(util.Base):
return_ids.append(rtrn["success"])
return all(return_ids)
@util.handle_exceptions
def create_ad(self, asset, countrycode, currency, provider, edit=False, ad_id=None):
def format_ad(self, asset, currency, payment_details_text):
"""
Post an ad with the given asset in a country with a given currency.
Convert the min and max amounts from settings to the given currency with CurrencyRates.
:param asset: the crypto asset to list (XMR or BTC)
:type asset: string
:param countrycode: country code
:param currency: currency code
:type countrycode: string
:type currency: string
:return: data about created object or error
:rtype: dict
Format the ad.
"""
ad = settings.Agora.Ad
paymentdetails = settings.Agora.PaymentDetails
# Substitute the currency
ad = ad.replace("$CURRENCY$", currency)
if currency == "GBP":
ad = ad.replace("$PAYMENT$", settings.Agora.GBPDetailsAd)
paymentdetailstext = paymentdetails.replace("$PAYMENT$", settings.Agora.GBPDetailsPayment)
else:
ad = ad.replace("$PAYMENT$", settings.Agora.DefaultDetailsAd)
paymentdetailstext = paymentdetails.replace("$PAYMENT$", settings.Agora.DefaultDetailsPayment)
# Substitute the asset
ad = ad.replace("$ASSET$", asset)
# Substitute the payment details
ad = ad.replace("$PAYMENT$", payment_details_text)
# Strip extra tabs
ad = ad.replace("\\t", "\t")
return ad
def format_payment_details(self, currency, payment_details):
"""
Format the payment details.
"""
payment = settings.Agora.PaymentDetails
payment_text = ""
for field, value in payment_details.items():
formatted_name = field.replace("_", " ")
formatted_name = formatted_name.capitalize()
payment_text += f"* {formatted_name}: **{value}**"
if field != list(payment_details.keys())[-1]: # No trailing newline
payment_text += "\n"
payment = payment.replace("$PAYMENT$", payment_text)
payment = payment.replace("$CURRENCY$", currency)
return payment
def get_minmax(self, asset, currency):
rates = self.money.get_rates_all()
if currency not in rates and not currency == "USD":
self.log.error(f"Can't create ad without rates: {currency}")
return
if asset == "XMR":
min_usd = float(settings.Agora.MinUSDXMR)
max_usd = float(settings.Agora.MaxUSDXMR)
@ -496,9 +509,33 @@ class Agora(util.Base):
else:
min_amount = rates[currency] * min_usd
max_amount = rates[currency] * max_usd
return (min_amount, max_amount)
@util.handle_exceptions
def create_ad(self, asset, countrycode, currency, provider, payment_details, visible=True, edit=False, ad_id=None):
"""
Post an ad with the given asset in a country with a given currency.
Convert the min and max amounts from settings to the given currency with CurrencyRates.
:param asset: the crypto asset to list (XMR or BTC)
:type asset: string
:param countrycode: country code
:param currency: currency code
:param payment_details: the payment details
:type countrycode: string
:type currency: string
:type payment_details: dict
:return: data about created object or error
:rtype: dict
"""
if payment_details:
payment_details_text = self.format_payment_details(currency, payment_details)
ad_text = self.format_ad(asset, currency, payment_details_text)
min_amount, max_amount = self.get_minmax(asset, currency)
price_formula = f"coingecko{asset.lower()}usd*usd{currency.lower()}*{settings.Agora.Margin}"
# Remove extra tabs
ad = ad.replace("\\t", "\t")
form = {
"country_code": countrycode,
"currency": currency,
@ -508,12 +545,15 @@ class Agora(util.Base):
"track_max_amount": False,
"require_trusted_by_advertiser": False,
"online_provider": provider,
"msg": ad,
"min_amount": min_amount,
"max_amount": max_amount,
"payment_method_details": settings.Agora.PaymentMethodDetails,
"account_info": paymentdetailstext,
"visible": visible,
}
if payment_details:
form["account_info"] = payment_details_text
form["msg"] = ad_text
form["min_amount"] = min_amount
form["max_amount"] = max_amount
if edit:
ad = self.agora.ad(ad_id=ad_id, **form)
else:

View File

@ -505,7 +505,7 @@ class AgoraDesk:
if lon:
params["lon"] = lon
if visible:
params["visible"] = 1 if visible else 0
params["visible"] = True if visible else False
return self._api_call(
api_method=f"ad/{ad_id}",

View File

@ -165,3 +165,48 @@ class Markets(util.Base):
if filter_asset:
if asset == filter_asset:
yield (asset, countrycode, currency, provider)
def distribute_account_details(self, currencies=None, account_info=None):
"""
Distribute account details for ads.
We will disable ads we can't support.
"""
if not currencies:
currencies = self.sinks.currencies
if not account_info:
account_info = self.sinks.account_info
# First, let's get the ads we can't support
all_currencies = self.get_all_currencies()
supported_currencies = [currency for currency in currencies if currency in all_currencies]
# not_supported = [currency for currency in all_currencies if currency not in supported_currencies]
our_ads = self.agora.enum_ads()
supported_ads = [ad for ad in our_ads if ad[3] in supported_currencies]
not_supported_ads = [ad for ad in our_ads if ad[3] not in supported_currencies]
currency_account_info_map = {}
for currency in supported_currencies:
for bank, accounts in account_info.items():
for account in accounts:
if account["currency"] == currency:
currency_account_info_map[currency] = account["account_number"]
for ad in supported_ads:
asset = ad[0]
countrycode = ad[2]
currency = ad[3]
provider = ad[4]
payment_details = currency_account_info_map[currency]
ad_id = ad[1]
self.agora.create_ad(asset, countrycode, currency, provider, payment_details, visible=True, edit=True, ad_id=ad_id)
for ad in not_supported_ads:
asset = ad[0]
countrycode = ad[2]
currency = ad[3]
provider = ad[4]
ad_id = ad[1]
self.agora.create_ad(asset, countrycode, currency, provider, payment_details=False, visible=False, edit=True, ad_id=ad_id)

View File

@ -18,23 +18,61 @@ class Sinks(util.Base):
def __init__(self):
super().__init__()
self.account_info = {}
def __irc_started__(self):
self.startup()
def startup(self):
"""
We NEED the other libraries, and we initialise fast, so don't make
any race conditions by relying on something that might not be there.
"""
self.fidor = sinks.fidor.Fidor()
self.nordigen = sinks.nordigen.Nordigen()
self.truelayer = sinks.truelayer.TrueLayer(self)
# setattr(self.truelayer, "sinks", self)
def got_transactions(self, bank, account_id, transactions):
print("GOT transactions", bank, account_id, transactions)
if not transactions:
return False
transaction_ids = [x["transaction_id"] for x in transactions]
print("IDS", transaction_ids)
new_key_name = f"new.transactions.{bank}.{account_id}"
old_key_name = f"transactions.{bank}.{account_id}"
r.sset(new_key_name, transaction_ids)
# for transaction_id in transaction_ids:
if not transaction_ids:
return
r.sadd(new_key_name, *transaction_ids)
difference = r.sdiff(new_key_name, old_key_name)
print("difference", difference)
difference = list(r.sdiff(new_key_name, old_key_name))
difference = util.convert(difference)
new_transactions = [x for x in transactions if x["transaction_id"] in difference]
# Rename the new key to the old key so we can run the diff again
r.rename(new_key_name, old_key_name)
for transaction in new_transactions:
self.tx.transaction(transaction)
# self.transactions.transaction(transactions)
def got_account_info(self, subclass, account_infos):
"""
Called when we get account information from an API provider.
:param subclass: class name that called it, truelayer, fidor, etc
:param account_infos: dict of dicts of account information
:param account_infos: dict
"""
for bank, accounts in account_infos.items():
for account in list(accounts):
if len(account["account_number"]) == 1:
account_infos[bank].remove(account)
currencies = [account["currency"] for bank, accounts in account_infos.items() for account in accounts]
self.account_info = account_infos
self.currencies = currencies
# parsed_details =
# {"EUR": {"IBAN": "xxx", "BIC": "xxx"},
# "GBP": {"SORT": "04-04-04", "ACCOUNT": "1922-2993"}}
# self.markets.distribute_account_details(currencies, account_infos)

View File

@ -23,6 +23,8 @@ class TrueLayer(util.Base):
self.sinks = sinks
self.tokens = {}
self.banks = {}
self.refresh_tokens = {}
self.authed = False
# Get the banks from the config and cache them
self.get_mapped_accounts()
@ -35,6 +37,15 @@ class TrueLayer(util.Base):
# -> set self.tokens[bank] = access_token
self.lc.start(int(settings.TrueLayer.TokenRefreshSec))
def __authed__(self):
"""
Called when we have received all the API tokens.
"""
# Get the account information and pass it to the main function
self.log.info("All accounts authenticated: " + ", ".join(self.tokens.keys()))
account_infos = self.get_all_account_info()
self.sinks.got_account_info("truelayer", account_infos)
self.lc_tx = LoopingCall(self.transaction_loop)
self.lc_tx.start(int(settings.TrueLayer.RefreshSec))
@ -123,8 +134,12 @@ class TrueLayer(util.Base):
refresh_tokens = loads(settings.TrueLayer.RefreshKeys)
# Set the cached entry
self.refresh_tokens = refresh_tokens
for bank in refresh_tokens:
self.get_new_token(bank)
rtrn = self.get_new_token(bank)
if not rtrn:
self.log.error(f"Error getting token for {bank}")
return
def get_new_token(self, bank):
"""
@ -133,6 +148,7 @@ class TrueLayer(util.Base):
:type account:
"""
if bank not in self.refresh_tokens:
self.log.error(f"Bank {bank} not in refresh tokens")
return
headers = {"Content-Type": "application/x-www-form-urlencoded"}
@ -146,17 +162,22 @@ class TrueLayer(util.Base):
try:
parsed = r.json()
except JSONDecodeError:
self.log.error(f"Failed to decode JSON: {r.content}")
return False
if r.status_code == 200:
if "access_token" in parsed.keys():
self.tokens[bank] = parsed["access_token"]
self.log.info(f"Refreshed access token for {bank}")
# self.log.info(f"Refreshed access token for {bank}")
if len(self.refresh_tokens.keys()) == len(self.tokens.keys()) and not self.authed:
# We are now fully authenticated and ready to start loops!
self.__authed__()
self.authed = True
return True
else:
self.log.error(f"Token refresh didn't contain access token: {parsed}", parsed=parsed)
self.log.error(f"Token refresh didn't contain access token: {parsed}")
return False
else:
self.log.error(f"Cannot refresh token: {parsed}", parsed=parsed)
self.log.error(f"Cannot refresh token: {parsed}")
return False
def get_accounts(self, bank):
@ -183,7 +204,7 @@ class TrueLayer(util.Base):
try:
parsed = r.json()
except JSONDecodeError:
self.log.error("Error parsing accounts response: {content}", content=r.content)
self.log.error(f"Error parsing accounts response: {r.content}")
return False
return parsed
@ -192,6 +213,17 @@ class TrueLayer(util.Base):
existing_entry = loads(settings.TrueLayer.Maps)
self.banks = existing_entry
def get_all_account_info(self):
to_return = {}
for bank in self.banks:
for account_id in self.banks[bank]:
account_data = self.get_account(bank, account_id)
if bank in to_return:
to_return[bank].append(account_data)
else:
to_return[bank] = [account_data]
return to_return
def get_account(self, bank, account_id):
account_data = self._get_account(bank, account_id)
if "results" not in account_data:
@ -241,7 +273,9 @@ class TrueLayer(util.Base):
try:
parsed = r.json()
except JSONDecodeError:
self.log.error("Error parsing transactions response: {content}", content=r.content)
self.log.error(f"Error parsing transactions response: {r.content}")
return False
if "results" in parsed:
return parsed["results"]
else:
return False
return parsed["results"]

View File

@ -229,3 +229,32 @@ class TestAgora(TestCase):
# Test specifying rates=
lookup_rates_return = self.agora.money.lookup_rates(enum_ads_return, rates=cg_prices)
self.assertCountEqual(lookup_rates_return, expected_return)
def test_format_ad(self):
settings.settings.Agora.Ad = """* Set **Country of recipient's bank** to **"United Kingdom"**
$PAYMENT$
* Set **Company name** to **"PATHOGEN LIMITED"**"""
payment_details = {"sort_code": "02-03-04", "account_number": "0023-0045"}
payment_details_text = self.agora.format_payment_details("GBP", payment_details)
ad_text = self.agora.format_ad("XMR", "GBP", payment_details_text)
expected = """* Set **Country of recipient's bank** to **"United Kingdom"**
* Company name: **PATHOGEN LIMITED**
* Sort code: **02-03-04**
* Account number: **0023-0045**
* Please send in **GBP**
* If you are asked for address information, please use **24 Holborn Viaduct, London, England, EC1A 2BN**
* The post code is **EC1A 2BN**
* Set **Company name** to **"PATHOGEN LIMITED"**"""
self.assertEqual(ad_text, expected)
def test_format_payment_details(self):
payment_details = {"sort_code": "02-03-04", "account_number": "0023-0045"}
payment_details_text = self.agora.format_payment_details("GBP", payment_details)
expected = """* Company name: **PATHOGEN LIMITED**
* Sort code: **02-03-04**
* Account number: **0023-0045**
* Please send in **GBP**
* If you are asked for address information, please use **24 Holborn Viaduct, London, England, EC1A 2BN**
* The post code is **EC1A 2BN**"""
self.assertEqual(payment_details_text, expected)

View File

@ -75,92 +75,29 @@ class Transactions(util.Base):
:param data: details of transaction
:type data: dict
"""
event = data["event"]
ts = data["timestamp"]
if "data" not in data:
return
inside = data["data"]
txid = inside["id"]
if "type" not in inside:
# stored_trade here is actually TX
stored_trade = r.hgetall(f"tx.{txid}")
if not stored_trade:
self.log.error(f"Could not find entry in DB for typeless transaction: {txid}")
return
stored_trade = util.convert(stored_trade)
if "old_state" in inside:
if "new_state" in inside:
# We don't care unless we're being told a transaction is now completed
if not inside["new_state"] == "completed":
return
# We don't care unless the existing trade is pending
if not stored_trade["state"] == "pending":
return
# Check the old state is what we also think it is
if inside["old_state"] == stored_trade["state"]:
# Set the state to the new state
stored_trade["state"] = inside["new_state"]
# Store the updated state
r.hmset(f"tx.{txid}", stored_trade)
# Check it's all been previously validated
if "valid" not in stored_trade:
self.log.error(f"Valid not in stored trade for {txid}, aborting.")
return
if stored_trade["valid"] == "1":
# Make it invalid immediately, as we're going to release now
stored_trade["valid"] = "0"
r.hmset(f"tx.{txid}", stored_trade)
reference = self.tx_to_ref(stored_trade["trade_id"])
self.release_funds(stored_trade["trade_id"], reference)
self.ux.notify.notify_complete_trade(stored_trade["amount"], stored_trade["currency"])
return
# If type not in inside and we haven't hit any more returns
return
else:
txtype = inside["type"]
if txtype == "card_payment":
self.log.info(f"Ignoring card payment: {txid}")
return
state = inside["state"]
if "reference" in inside:
reference = inside["reference"]
else:
reference = "not_given"
leg = inside["legs"][0]
if "counterparty" in leg:
account_type = leg["counterparty"]["account_type"]
else:
account_type = "not_given"
amount = leg["amount"]
txid = data["transaction_id"]
txtype = data["transaction_type"]
amount = data["amount"]
if amount <= 0:
self.log.info(f"Ignoring transaction with negative/zero amount: {txid}")
return
currency = leg["currency"]
description = leg["description"]
currency = data["currency"]
description = data["description"]
reference = data["meta"]["provider_reference"]
to_store = {
"event": event,
"trade_id": "",
"ts": ts,
"txid": txid,
"txtype": txtype,
"state": state,
"reference": reference,
"account_type": account_type,
"amount": amount,
"currency": currency,
"description": description,
"valid": 0, # All checks passed and we can release escrow?
}
self.log.info(f"Transaction processed: {dumps(to_store, indent=2)}")
self.irc.sendmsg(f"AUTO Incoming transaction: {amount}{currency} ({reference}) - {state} - {description}")
self.irc.sendmsg(f"AUTO Incoming transaction: {amount}{currency} ({reference}) - {description}")
# Partial reference implementation
# Account for silly people not removing the default string
# Split the reference into parts
@ -234,19 +171,9 @@ class Transactions(util.Base):
self.irc.sendmsg(f"Amount mismatch - not in margins: {stored_trade['amount']} (min: {min_amount} / max: {max_amount}")
return
# We have made it this far without hitting any of the returns, so let's set valid = True
# This will let us instantly release if the type is pending, and it is subsequently updated to completed with a callback.
to_store["valid"] = 1
# Store the trade ID so we can release it easily
to_store["trade_id"] = stored_trade["id"]
if not state == "completed":
self.log.info(f"Storing incomplete trade: {txid}")
r.hmset(f"tx.{txid}", to_store)
# Don't procees further if state is not "completed"
return
r.hmset(f"tx.{txid}", to_store)
self.release_funds(stored_trade["id"], stored_trade["reference"])
# self.release_funds(stored_trade["id"], stored_trade["reference"])
print("WOULD RELEASE THE FUCKING MONEY")
self.ux.notify.notify_complete_trade(amount, currency)
def release_funds(self, trade_id, reference):

View File

@ -477,7 +477,6 @@ class IRCCommands(object):
account_id = spl[2]
transactions = tx.sinks.truelayer.get_transactions(account, account_id)
for transaction in transactions:
print(transaction)
txid = transaction["transaction_id"]
ptxid = transaction["provider_transaction_id"]
txtype = transaction["transaction_type"]
@ -502,3 +501,31 @@ class IRCCommands(object):
msg(f"Failed to map the account")
return
msg(f"Mapped account ID {account_id} at bank {bank} to {account_name}")
class unmapped(object):
name = "unmapped"
authed = True
helptext = "Get unmapped accounts for a bank. Usage: unmapped <bank>"
@staticmethod
def run(cmd, spl, length, authed, msg, agora, tx, ux):
if length == 2:
bank = spl[1]
accounts_active = []
for bank, accounts in tx.sinks.truelayer.banks.items():
for account in accounts:
accounts_active.append(account)
accounts_all = tx.sinks.truelayer.get_accounts(bank)
accounts_unmapped = [x["account_id"] for x in accounts_all["results"] if x["account_id"] not in accounts_active]
msg(f"Unmapped accounts: {', '.join(accounts_unmapped)}")
class distdetails(object):
name = "distdetails"
authed = True
helptext = "Distribute account details among all ads."
@staticmethod
def run(cmd, spl, length, authed, msg, agora, tx, ux):
currencies = tx.sinks.currencies
tx.markets.distribute_account_details()
msg(f"Distributing account details for currencies: {', '.join(currencies)}")

View File

@ -118,6 +118,7 @@ class IRCBot(irc.IRCClient):
:type channel: string
"""
self.agora.setup_loop()
self.sinks.__irc_started__()
self.log.info(f"Joined channel {channel}")
def privmsg(self, user, channel, msg):