Implement transaction handling

This commit is contained in:
2023-03-13 18:49:47 +00:00
parent 780adf3bc1
commit 7d1bd75f48
12 changed files with 1098 additions and 318 deletions

View File

@@ -1,10 +1,8 @@
from abc import ABC
import orjson
from core.lib import db, notify
# from core.lib.money import money
from core.clients.platforms.agora import AgoraClient
from core.lib import notify
from core.lib.money import money
from core.util import logs
log = logs.get_logger("aggregator")
@@ -53,29 +51,55 @@ class AggregatorClient(ABC):
async def process_transactions(self, account_id, transactions):
if not transactions:
return False
transaction_ids = [x["transaction_id"] for x in transactions]
new_key_name = f"new.transactions.{self.instance.id}.{self.name}.{account_id}"
old_key_name = f"transactions.{self.instance.id}.{self.name}.{account_id}"
# for transaction_id in transaction_ids:
if not transaction_ids:
return
await db.r.sadd(new_key_name, *transaction_ids)
difference = list(await db.r.sdiff(new_key_name, old_key_name))
platforms = self.platforms
for transaction in transactions:
transaction_id = transaction["transaction_id"]
tx_obj = self.instance.get_transaction(
account_id,
transaction_id,
)
if tx_obj is None:
tx_cast = {
"transaction_id": transaction_id,
"recipient": transaction["creditorName"],
"sender": transaction["debtorName"],
"amount": transaction["amount"],
"currency": transaction["currency"],
"note": transaction["reference"],
}
tx_obj = self.instance.add_transaction(
account_id,
tx_cast,
)
# New transaction
await self.transaction(platforms, tx_obj)
else:
# Transaction exists
continue
# transaction_ids = [x["transaction_id"] for x in transactions]
# new_key_name = f"new.transactions.{self.instance.id}.{self.name}.{account_id}"
# old_key_name = f"transactions.{self.instance.id}.{self.name}.{account_id}"
# # for transaction_id in transaction_ids:
# if not transaction_ids:
# return
# await db.r.sadd(new_key_name, *transaction_ids)
difference = db.convert(difference)
# difference = list(await db.r.sdiff(new_key_name, old_key_name))
new_transactions = [
x for x in transactions if x["transaction_id"] in difference
]
# difference = db.convert(difference)
# Rename the new key to the old key so we can run the diff again
await db.r.rename(new_key_name, old_key_name)
for transaction in new_transactions:
transaction["subclass"] = self.name
# self.tx.transaction(transaction)
# new_transactions = [
# x for x in transactions if x["transaction_id"] in difference
# ]
def valid_transaction(self, data):
# # Rename the new key to the old key so we can run the diff again
# await db.r.rename(new_key_name, old_key_name)
# for transaction in new_transactions:
# transaction["subclass"] = self.name
# # self.tx.transaction(transaction)
def valid_transaction(self, tx_obj):
"""
Determine if a given transaction object is valid.
:param data: a transaction cast
@@ -83,56 +107,58 @@ class AggregatorClient(ABC):
:return: whether the transaction is valid
:rtype: bool
"""
txid = data["transaction_id"]
if "amount" not in data:
txid = tx_obj.transaction_id
if tx_obj.amount is None:
return False
if "currency" not in data:
if tx_obj.currency is None:
return False
amount = data["amount"]
amount = tx_obj.amount
if amount <= 0:
log.info(f"Ignoring transaction with negative/zero amount: {txid}")
return False
return True
def extract_reference(self, data):
"""
Extract a reference from the transaction cast.
:param data: a transaction cast
:type data: dict
:return: the extracted reference or not_set
:rtype: str
"""
if "reference" in data:
return data["reference"]
elif "meta" in data:
if "provider_reference" in data["meta"]:
return data["meta"]["provider_reference"]
return "not_set"
# def extract_reference(self, data):
# """
# Extract a reference from the transaction cast.
# :param data: a transaction cast
# :type data: dict
# :return: the extracted reference or not_set
# :rtype: str
# """
# if "reference" in data:
# return data["reference"]
# elif "meta" in data:
# if "provider_reference" in data["meta"]:
# return data["meta"]["provider_reference"]
# return "not_set"
def extract_sender(self, data):
"""
Extract a sender name from the transaction cast.
:param data: a transaction cast
:type data: dict
:return: the sender name or not_set
:rtype: str
"""
if "debtorName" in data:
return data["debtorName"]
elif "meta" in data:
if "debtor_account_name" in data["meta"]:
return data["meta"]["debtor_account_name"]
elif " " in data["reference"]:
refsplit = data["reference"].split(" ")
if not len(refsplit) == 2:
log.error(f"Sender cannot be extracted: {data}")
return "not_set"
realname, part2 = data["reference"].split(" ")
return realname
# def extract_sender(self, data):
# """
# Extract a sender name from the transaction cast.
# :param data: a transaction cast
# :type data: dict
# :return: the sender name or not_set
# :rtype: str
# """
# if "debtorName" in data:
# return data["debtorName"]
# elif "meta" in data:
# if "debtor_account_name" in data["meta"]:
# return data["meta"]["debtor_account_name"]
# elif " " in data["reference"]:
# refsplit = data["reference"].split(" ")
# if not len(refsplit) == 2:
# log.error(f"Sender cannot be extracted: {data}")
# return "not_set"
# realname, part2 = data["reference"].split(" ")
# return realname
return "not_set"
# return "not_set"
async def reference_partial_check(self, reference, txid, currency, amount):
async def reference_partial_check(
self, platform, reference, txid, currency, amount
):
"""
Perform a partial check by intersecting all parts of the split of the
reference against the existing references, and returning a set of the matches.
@@ -146,9 +172,10 @@ class AggregatorClient(ABC):
# Split the reference into parts
ref_split = reference.split(" ")
# Get all existing references
existing_refs = await db.get_refs()
existing_refs = platform.references
# Get all parts of the given reference split that match the existing references
stored_trade_reference = set(existing_refs).intersection(set(ref_split))
# stored_trade_reference = set(existing_refs).intersection(set(ref_split))
stored_trade_reference = [x for x in existing_refs if x in ref_split]
if len(stored_trade_reference) > 1:
message = (
f"Multiple references valid for TXID {txid}: {reference}"
@@ -162,38 +189,59 @@ class AggregatorClient(ABC):
return stored_trade_reference.pop()
# TODO: pass platform here
# async def can_alt_lookup(self, amount, currency, reference):
# amount_usd = self.money.to_usd(amount, currency)
# # Amount is reliable here as it is checked by find_trade,
# # so no need for stored_trade["amount"]
# if float(amount_usd) > float(settings.Agora.AcceptableAltLookupUSD):
# message = (
# f"Amount exceeds max for {reference}"
# f"Currency: {currency} | Amount: {amount}"
# )
# title = "Amount exceeds max for {reference}"
# await notify.sendmsg(self.instance.user, message, title=title)
# return False
# return True
async def can_alt_lookup(self, platform, amount, currency, reference):
amount_usd = await money.to_usd(amount, currency)
# Amount is reliable here as it is checked by find_trade,
# so no need for stored_trade["amount"]
if amount_usd > platform.no_reference_amount_check_max_usd:
message = (
f"Amount exceeds max for {reference}"
f"Currency: {currency} | Amount: {amount}"
)
title = "Amount exceeds max for {reference}"
await notify.sendmsg(self.instance.user, message, title=title)
return False
return True
async def amount_currency_lookup(self, amount, currency, txid, reference):
log.info(f"No reference in DB refs for {reference}")
self.irc.sendmsg(f"No reference in DB refs for {reference}")
# Try checking just amount and currency, as some people
# (usually people buying small amounts)
# are unable to put in a reference properly.
def find_trade(self, platform, txid, currency, amount):
"""
Get a trade reference that matches the given currency and amount.
Only works if there is one result.
:param txid: Sink transaction ID
:param currency: currency
:param amount: amount
:type txid: string
:type currency: string
:type amount: int
:return: matching trade object or False
:rtype: dict or bool
"""
refs = platform.references
matching_refs = []
# TODO: use get_ref_map in this function instead of calling get_ref multiple
# times
for ref in refs:
stored_trade = platform.get_trade_by_reference(ref)
if stored_trade.currency == currency and stored_trade.amount_fiat == amount:
matching_refs.append(stored_trade)
if len(matching_refs) != 1:
log.error(
f"Find trade returned multiple results for TXID {txid}: {matching_refs}"
)
return False
return matching_refs[0]
log.info(f"Checking against amount and currency for TXID {txid}")
self.irc.sendmsg(f"Checking against amount and currency for TXID {txid}")
async def amount_currency_lookup(self, platform, amount, currency, txid, ref):
title = f"Checking against amount and currency for TXID {txid}"
message = (
f"Checking against amount and currency for TXID {txid}"
f"Currency: {currency} | Amount: {amount}"
)
await notify.sendmsg(self.instance.user, message, title=title)
if not await self.can_alt_lookup(amount, currency, reference):
if not await self.can_alt_lookup(platform, amount, currency, ref):
return False
stored_trade = await self.find_trade(txid, currency, amount)
stored_trade = self.find_trade(platform, txid, currency, amount)
if not stored_trade:
title = f"Failed to get reference by amount and currency: {txid}"
message = (
@@ -202,11 +250,12 @@ class AggregatorClient(ABC):
)
await notify.sendmsg(self.instance.user, message, title=title)
return None
stored_trade["amount"] = float(stored_trade["amount"]) # convert to float
return stored_trade
async def normal_lookup(self, stored_trade_reference, reference, currency, amount):
stored_trade = await db.get_ref(stored_trade_reference)
async def normal_lookup(
self, platform, stored_trade_reference, reference, currency, amount
):
stored_trade = platform.get_trade_by_reference(stored_trade_reference)
if not stored_trade:
title = f"No reference in DB for {reference}"
message = (
@@ -215,27 +264,25 @@ class AggregatorClient(ABC):
)
await notify.sendmsg(self.instance.user, message, title=title)
return False
stored_trade["amount"] = float(stored_trade["amount"]) # convert to float
# stored_trade["amount"] = float(stored_trade["amount"]) # convert to float
return stored_trade
async def currency_check(self, currency, amount, reference, stored_trade):
if not stored_trade["currency"] == currency:
async def currency_check(self, currency, stored_trade):
if not stored_trade.currency == currency:
title = "Currency mismatch"
message = (
f"Currency mismatch, Agora: {stored_trade['currency']} "
f"Currency mismatch, Agora: {stored_trade.currency} "
f"/ Sink: {currency}"
)
await notify.sendmsg(self.instance.user, message, title=title)
return False
return True
async def alt_amount_check(
self, platform, amount, currency, reference, stored_trade
):
async def alt_amount_check(self, platform, amount, currency, stored_trade):
# If the amount does not match exactly, get the min and max values for our
# given acceptable margins for trades
min_amount, max_amount = await self.money.get_acceptable_margins(
platform, currency, stored_trade["amount"]
min_amount, max_amount = await money.get_acceptable_margins(
platform, currency, stored_trade.amount_fiat
)
log.info(
(
@@ -243,7 +290,6 @@ class AggregatorClient(ABC):
f" / max: {max_amount}"
)
)
self.irc.sendmsg()
title = "Amount does not match exactly"
message = (
f"Amount does not match exactly, trying with margins: min: "
@@ -253,14 +299,14 @@ class AggregatorClient(ABC):
if not min_amount < amount < max_amount:
title = "Amount mismatch - not in margins"
message = (
f"Amount mismatch - not in margins: {stored_trade['amount']} "
f"Amount mismatch - not in margins: {stored_trade.amount_fiat} "
f"(min: {min_amount} / max: {max_amount}"
)
await notify.sendmsg(self.instance.user, message, title=title)
return False
return True
async def transaction(self, data):
async def transaction(self, platforms, tx_obj):
"""
Store details of transaction and post notifications to IRC.
Matches it up with data stored in Redis to attempt to reconcile with an Agora
@@ -268,104 +314,93 @@ class AggregatorClient(ABC):
:param data: details of transaction
:type data: dict
"""
valid = self.valid_transaction(data)
valid = self.valid_transaction(tx_obj)
if not valid:
return False
ts = data["timestamp"]
txid = data["transaction_id"]
amount = float(data["amount"])
currency = data["currency"]
txid = tx_obj.transaction_id
amount = tx_obj.amount
currency = tx_obj.currency
reference = self.extract_reference(data)
sender = self.extract_sender(data)
reference = tx_obj.note
subclass = data["subclass"]
to_store = {
"subclass": subclass,
"ts": ts,
"txid": txid,
"reference": reference,
"amount": amount,
"currency": currency,
"sender": sender,
}
db.r.hmset(f"tx.{txid}", to_store)
# reference = self.extract_reference(data)
# sender = tx_obj.sender
log.info(f"Transaction processed: {orjson.dumps(to_store, indent=2)}")
self.irc.sendmsg(
(
f"AUTO Incoming transaction on {subclass}: {txid} {amount}{currency} "
f"({reference})"
)
log.info(f"Transaction processed: {tx_obj}")
await notify.sendmsg(
self.instance.user,
(f"Transaction: {txid} {amount}{currency}: {reference}"),
title="Incoming transaction",
)
stored_trade_reference = self.reference_partial_check(
reference, txid, currency, amount
)
if stored_trade_reference is False: # can be None though
return
stored_trade = False
looked_up_without_reference = False
# Normal implementation for when we have a reference
if stored_trade_reference:
stored_trade = self.normal_lookup(
stored_trade_reference, reference, currency, amount
for platform in platforms:
stored_trade_reference = await self.reference_partial_check(
platform, reference, txid, currency, amount
)
# if not stored_trade:
if stored_trade_reference is False: # can be None though
continue
stored_trade = False
looked_up_without_reference = False
# Normal implementation for when we have a reference
if stored_trade_reference:
stored_trade = await self.normal_lookup(
platform, stored_trade_reference, reference, currency, amount
)
# if not stored_trade:
# return
# Amount/currency lookup implementation for when we have no reference
else:
if not stored_trade: # check we don't overwrite the lookup above
stored_trade = await self.amount_currency_lookup(
platform, amount, currency, txid, reference
)
if stored_trade is False:
continue
if stored_trade:
# Note that we have looked it up without reference so we don't
# use +- below
# This might be redundant given the checks in find_trade,
# but better safe than sorry!
looked_up_without_reference = True
else:
continue
else:
# Stored trade reference is none, the checks below will do nothing
continue
# Make sure it was sent in the expected currency
if not await self.currency_check(currency, stored_trade):
continue
# Make sure the expected amount was sent
if not stored_trade.amount_fiat == amount:
if looked_up_without_reference:
continue
if not await self.alt_amount_check(
platform, amount, currency, stored_trade
):
continue
# platform_buyer = stored_trade["buyer"]
# Check sender - we don't do anything with this yet
# sender_valid = antifraud.check_valid_sender(
# reference, platform, sender, platform_buyer
# )
# log.info(f"Trade {reference} buyer {platform_buyer}
# valid: {sender_valid}")
instance = await AgoraClient(platform)
rtrn = await instance.release_map_trade(stored_trade, tx_obj)
# if trade_released:
# self.ux.notify.notify_complete_trade(amount, currency)
# else:
# log.error(f"Cannot release trade {reference}.")
# return
# Amount/currency lookup implementation for when we have no reference
else:
if not stored_trade: # check we don't overwrite the lookup above
stored_trade = self.amount_currency_lookup(
amount, currency, txid, reference
)
if stored_trade is False:
return
if stored_trade:
# Note that we have looked it up without reference so we don't use
# +- below
# This might be redundant given the amount checks in find_trade,
# but better safe than sorry!
looked_up_without_reference = True
else:
return
else:
# Stored trade reference is none, the checks below will do nothing
return
# Make sure it was sent in the expected currency
if not self.currency_check(currency, amount, reference, stored_trade):
return
# Make sure the expected amount was sent
if not stored_trade["amount"] == amount:
if looked_up_without_reference:
return
platform = stored_trade["subclass"]
if not self.alt_amount_check(
platform, amount, currency, reference, stored_trade
):
return
platform = stored_trade["subclass"]
# platform_buyer = stored_trade["buyer"]
# Check sender - we don't do anything with this yet
# sender_valid = antifraud.check_valid_sender(
# reference, platform, sender, platform_buyer
# )
# log.info(f"Trade {reference} buyer {platform_buyer} valid: {sender_valid}")
# trade_released = self.release_map_trade(reference, txid)
# if trade_released:
# self.ux.notify.notify_complete_trade(amount, currency)
# else:
# log.error(f"Cannot release trade {reference}.")
# return
rtrn = await self.release_funds(stored_trade["id"], stored_trade["reference"])
if rtrn:
title = "Trade complete"
message = f"Trade complete: {amount}{currency}"
await notify.sendmsg(self.instance.user, message, title=title)
# rtrn = await platform.release_funds(stored_trade["id"],
# stored_trade["reference"])
if rtrn:
title = "Trade complete"
message = f"Trade complete: {amount}{currency}"
await notify.sendmsg(self.instance.user, message, title=title)

View File

@@ -306,7 +306,7 @@ class NordigenClient(BaseClient, AggregatorClient):
self.normalise_transactions(parsed, state="booked")
if process:
await self.process_transactions(parsed)
await self.process_transactions(account_id, parsed)
if pending:
parsed_pending = response["pending"]
self.normalise_transactions(parsed_pending, state="pending")

View File

@@ -778,72 +778,68 @@ class LocalPlatformClient(ABC):
return all(actioned)
async def release_funds(self, trade_id, reference):
async def release_trade_escrow(self, trade_id, reference):
# stored_trade = await db.get_ref(reference)
logmessage = f"All checks passed, releasing funds for {trade_id} {reference}"
log.info(logmessage)
title = "Releasing escrow"
await notify.sendmsg(self.instance.user, logmessage, title=title)
release = self.release_funds
post_message = self.api.contact_message_post
rtrn = await release(trade_id)
if rtrn["message"] == "OK":
post_message(trade_id, "Thanks! Releasing now :)")
return True
else:
logmessage = f"Release funds unsuccessful: {rtrn['message']}"
title = "Release unsuccessful"
log.error(logmessage)
await notify.sendmsg(self.instance.user, logmessage, title=title)
return
# THIS IS NOT A COMMENT
# THIS IS FOR SECURITY
# WHEN IT HAS BEEN CONFIRMED TO WORK
# THIS CAN BE UNCOMMENTED
# rtrn = await self.release_funds(trade_id)
# if rtrn["message"] == "OK":
# await self.api.contact_message_post(trade_id, "Thanks! Releasing now :)")
# return True
# else:
# logmessage = f"Release funds unsuccessful: {rtrn['message']}"
# title = "Release unsuccessful"
# log.error(logmessage)
# await notify.sendmsg(self.instance.user, logmessage, title=title)
# return
# UNCOMMENT TO HERE
# # Parse the escrow release response
# message = rtrn["message"]
# # message_long = rtrn["response"]["data"]["message"]
# self.irc.sendmsg(f"{dumps(message)}")
async def update_trade_tx(self, reference, txid):
async def update_trade_tx(self, stored_trade, tx_obj):
"""
Update a trade to point to a given transaction ID.
Return False if the trade already has a mapped transaction.
Return False if the transaction already has a mapped trade.
"""
existing_tx = await db.r.hget(f"trade.{reference}", "tx")
if existing_tx is None:
return None
elif existing_tx == b"":
await db.r.hset(f"trade.{reference}", "tx", txid)
return True
else: # Already a mapped transaction
if tx_obj.reconciled:
return False
async def release_map_trade(self, reference, tx):
if tx_obj in stored_trade.linked.all():
return False
stored_trade.linked.add(tx_obj)
stored_trade.save()
tx_obj.reconciled = True
tx_obj.save()
return True
async def release_map_trade(self, stored_trade, tx_obj):
"""
Map a trade to a transaction and release if no other TX is
mapped to the same trade.
"""
stored_trade = await db.get_ref(reference)
if not stored_trade:
log.error(f"Could not get stored trade for {reference}.")
return None
tx_obj = await db.get_tx(tx)
if not tx_obj:
log.error(f"Could not get TX for {tx}.")
return None
platform_buyer = stored_trade["buyer"]
bank_sender = tx_obj["sender"]
trade_id = stored_trade["id"]
is_updated = await self.update_trade_tx(reference, tx)
if is_updated is None:
return None
elif is_updated is True:
platform_buyer = stored_trade.buyer
bank_sender = tx_obj.sender
trade_id = stored_trade.contact_id
is_updated = await self.update_trade_tx(stored_trade, tx_obj)
if is_updated is True:
# We mapped the trade successfully
self.release_funds(trade_id, reference)
antifraud.add_bank_sender(platform_buyer, bank_sender)
await self.release_trade_escrow(trade_id, stored_trade.reference)
await antifraud.add_bank_sender(platform_buyer, bank_sender)
return True
elif is_updated is False:
else:
# Already mapped
log.error(f"Trade {reference} already has a TX mapped, cannot map {tx}.")
log.error(
f"Trade {stored_trade} already has a TX mapped, cannot map {tx_obj}."
)
return False
async def new_trade(

View File

@@ -3,6 +3,9 @@ from pyotp import TOTP
from core.clients.base import BaseClient
from core.clients.platform import LocalPlatformClient
from core.util import logs
log = logs.get_logger("agora")
class AgoraClient(LocalPlatformClient, BaseClient):
@@ -20,15 +23,12 @@ class AgoraClient(LocalPlatformClient, BaseClient):
"""
print("CALLING RELEASE FUNDS", contact_id)
if self.instance.dummy:
self.log.error(
f"Running in dummy mode, not releasing funds for {contact_id}"
)
log.error(f"Running in dummy mode, not releasing funds for {contact_id}")
return
payload = {"tradeId": contact_id, "password": self.sets.Pass}
rtrn = await self.api._api_call(
api_method=f"contact_release/{contact_id}",
http_method="POST",
query_values=payload,
rtrn = await self.api.contact_release(
contact_id,
self.instance.password,
)
# Check if we can withdraw funds

View File

@@ -88,7 +88,6 @@ class AgoraDesk:
async with session.post(api_call_url, **cast) as response_raw:
response = await response_raw.json()
status_code = response_raw.status
else:
cast["params"] = query_values
async with aiohttp.ClientSession() as session:
@@ -226,11 +225,23 @@ class AgoraDesk:
)
# Todo:
# post/trade/contact_release/{trade_id} • Release trade escrow
# post/contact_fund/{trade_id} • Fund a trade
# post/contact_dispute/{trade_id} • Start a trade dispute
# post/contact_mark_as_paid/{trade_id} • Mark a trade as paid
async def contact_release(self, trade_id: str, password: str) -> Dict[str, Any]:
"""See Agoradesk API documentation.
https://agoradesk.com/api-docs/v1#operation/releaseEscrow
"""
payload = {"tradeId": trade_id, "password": password}
return await self._api_call(
api_method=f"contact_release/{trade_id}",
http_method="POST",
query_values=payload,
)
async def contact_mark_as_paid(self, trade_id: str) -> Dict[str, Any]:
"""See Agoradesk API.