From 1040579d3754dbd7671b193003c141e3ce4f73e6 Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 5 May 2022 18:17:24 +0100 Subject: [PATCH] Cleanup old libraries --- handler/markets.py | 317 --------------- handler/money.py | 148 ------- handler/transactions.py | 881 ---------------------------------------- 3 files changed, 1346 deletions(-) delete mode 100644 handler/markets.py delete mode 100644 handler/money.py delete mode 100644 handler/transactions.py diff --git a/handler/markets.py b/handler/markets.py deleted file mode 100644 index 219288d..0000000 --- a/handler/markets.py +++ /dev/null @@ -1,317 +0,0 @@ -# Other library imports -from json import loads - -# Project imports -from settings import settings -import util - - -class Markets(util.Base): - """ " - Markets handler for generic market functions. - """ - - def get_all_assets(self, platform): - sets = util.get_settings(platform) - assets = loads(sets.AssetList) - return assets - - def get_all_providers(self, platform): - sets = util.get_settings(platform) - providers = loads(sets.ProviderList) - return providers - - def get_all_currencies(self, platform): - sets = util.get_settings(platform) - currencies = list(set([x[0] for x in loads(sets.DistList)])) - return currencies - - def get_new_ad_equations(self, platform, public_ads, assets=None): - """ - Update all our prices. - :param public_ads: dictionary of public ads keyed by currency - :type public_ads: dict - :return: list of ads to modify - :rtype: list - """ - sets = util.get_settings(platform) - username = sets.Username - min_margin = sets.MinMargin - max_margin = sets.MaxMargin - - to_update = [] - - # NOTES: - # Get all ads for each currency, with all the payment methods. - # Create a function to, in turn, filter these so it contains only one payment method. Run autoprice on this. - # Append all results to to_update. Repeat for remaining payment methods, then call slow update. - - # (asset, currency, provider) - if not assets: - assets = self.get_all_assets(platform) - currencies = self.get_all_currencies(platform) - providers = self.get_all_providers(platform) - if platform == "lbtc": - providers = [self.sources.lbtc.map_provider(x, reverse=True) for x in providers] - sinks_currencies = self.sinks.currencies - supported_currencies = [currency for currency in currencies if currency in sinks_currencies] - currencies = supported_currencies - - brute = [(asset, currency, provider) for asset in assets for currency in currencies for provider in providers] - for asset, currency, provider in brute: - # Filter currency - try: - public_ads_currency = public_ads[currency] - except KeyError: - # self.log.error("Error getting public ads for currency {currency}", currency=currency) - if currency == "GBP": - self.log.error("Error getting public ads for currency USD, aborting") - break - continue - - # Filter asset - public_ads_filtered = [ad for ad in public_ads_currency if ad[4] == asset] - # Filter provider - public_ads_filtered = [ad for ad in public_ads_filtered if ad[3] == provider] - our_ads = [ad for ad in public_ads_filtered if ad[1] == username] - if not our_ads: - self.log.warning(f"No ads found in {platform} public listing for {asset} {currency} {provider}") - continue - new_margin = self.autoprice(username, min_margin, max_margin, public_ads_filtered, currency) - # self.log.info("New rate for {currency}: {rate}", currency=currency, rate=new_margin) - if platform == "agora": - new_formula = f"coingecko{asset.lower()}usd*usd{currency.lower()}*{new_margin}" - elif platform == "lbtc": - new_formula = f"btc_in_usd*{new_margin}*USD_in_{currency}" - for ad in our_ads: - ad_id = ad[0] - asset = ad[4] - our_margin = ad[5] - if new_margin != our_margin: - to_update.append([ad_id, new_formula, asset, currency, False]) - - return to_update - - def autoprice(self, username, min_margin, max_margin, ads, currency): - """ - Helper function to automatically adjust the price up/down in certain markets - in order to gain the most profits and sales. - :param ads: list of ads - :type ads: list of lists - :param currency: currency of the ads - :type currency: string - :return: the rate we should use for this currency - :rtype: float - """ - # self.log.debug("Autoprice starting for {x}", x=currency) - # Find cheapest ad - # Filter by 3rd index on each ad list to find the cheapest - min_margin_ad = min(ads, key=lambda x: x[6]) - # self.log.debug("Minimum margin ad: {x}", x=min_margin_ad) - - # Find second cheapest that is not us - # Remove results from ads that are us - ads_without_us = [ad for ad in ads if not ad[1] == username] - # self.log.debug("Ads without us: {x}", x=ads_without_us) - # Find ads above our min that are not us - ads_above_our_min_not_us = [ad for ad in ads_without_us if ad[6] > float(min_margin)] - # self.log.debug("Ads above our min not us: {x}", x=ads_above_our_min_not_us) - # Check that this list without us is not empty - if ads_without_us: - # Find the cheapest from these - min_margin_ad_not_us = min(ads_without_us, key=lambda x: x[6]) - # self.log.debug("Min margin ad not us: {x}", x=min_margin_ad_not_us) - # Lowball the lowest ad that is not ours - lowball_lowest_not_ours = min_margin_ad_not_us[6] # - 0.005 - # self.log.debug("Lowball lowest not ours: {x}", x=lowball_lowest_not_ours) - - # Check if the username field of the cheapest ad matches ours - if min_margin_ad[1] == username: - # self.log.debug("We are the cheapest for: {x}", x=currency) - # We are the cheapest! - # Are all of the ads ours? - all_ads_ours = all([ad[1] == username for ad in ads]) - if all_ads_ours: - # self.log.debug("All ads are ours for: {x}", x=currency) - # Now we know it's safe to return the maximum value - return float(max_margin) - else: - # self.log.debug("All ads are NOT ours for: {x}", x=currency) - # All the ads are not ours, but we are first... - # Check if the lowballed, lowest (that is not ours) ad's margin - # is less than our minimum - if lowball_lowest_not_ours < float(min_margin): - # self.log.debug("Lowball lowest not ours less than MinMargin") - return float(min_margin) - elif lowball_lowest_not_ours > float(max_margin): - # self.log.debug("Lowball lowest not ours more than MaxMargin") - return float(max_margin) - else: - # self.log.debug("Returning lowballed figure: {x}", x=lowball_lowest_not_ours) - return lowball_lowest_not_ours - else: - # self.log.debug("We are NOT the cheapest for: {x}", x=currency) - # We are not the cheapest :( - # Check if this list is empty - if not ads_above_our_min_not_us: - # Return the maximum margin? - return float(max_margin) - # Find cheapest ad above our min that is not us - cheapest_ad = min(ads_above_our_min_not_us, key=lambda x: x[4]) - cheapest_ad_margin = cheapest_ad[6] # - 0.005 - if cheapest_ad_margin > float(max_margin): - # self.log.debug("Cheapest ad not ours more than MaxMargin") - return float(max_margin) - # self.log.debug("Cheapest ad above our min that is not us: {x}", x=cheapest_ad) - return cheapest_ad_margin - - def create_distribution_list(self, platform, filter_asset=None): - """ - Create a list for distribution of ads. - :return: generator of asset, countrycode, currency, provider - :rtype: generator of tuples - """ - sets = util.get_settings(platform) - - # Iterate providers like REVOLUT, NATIONAL_BANK - for provider in loads(sets.ProviderList): - # Iterate assets like XMR, BTC - for asset in loads(sets.AssetList): - # Iterate pairs of currency and country like EUR, GB - for currency, countrycode in loads(sets.DistList): - if filter_asset: - if asset == filter_asset: - yield (asset, countrycode, currency, provider) - else: - yield (asset, countrycode, currency, provider) - - def get_valid_account_details(self, platform): - currencies = self.sinks.currencies - account_info = self.sinks.account_info - all_currencies = self.get_all_currencies(platform) - supported_currencies = [currency for currency in currencies if currency in all_currencies] - currency_account_info_map = {} - for currency in supported_currencies: - for bank, accounts in account_info.items(): - for account in accounts: - if account["currency"] == currency: - currency_account_info_map[currency] = account["account_number"] - currency_account_info_map[currency]["bank"] = bank.split("_")[0] - currency_account_info_map[currency]["recipient"] = account["recipient"] - return (supported_currencies, currency_account_info_map) - - def get_matching_account_details(self, platform, currency): - supported_currencies, currency_account_info_map = self.get_valid_account_details(platform) - if currency not in supported_currencies: - return False - return currency_account_info_map[currency] - - def _distribute_account_details(self, platform, currencies=None, account_info=None): - """ - Distribute account details for ads. - We will disable ads we can't support. - """ - if platform == "agora": - caller = self.agora - elif platform == "lbtc": - caller = self.lbtc - - if not currencies: - currencies = self.sinks.currencies - if not account_info: - account_info = self.sinks.account_info - supported_currencies, currency_account_info_map = self.get_valid_account_details(platform) - - # not_supported = [currency for currency in all_currencies if currency not in supported_currencies] - - our_ads = caller.enum_ads() - - supported_ads = [ad for ad in our_ads if ad[3] in supported_currencies] - - not_supported_ads = [ad for ad in our_ads if ad[3] not in supported_currencies] - - for ad in supported_ads: - asset = ad[0] - countrycode = ad[2] - currency = ad[3] - provider = ad[4] - payment_details = currency_account_info_map[currency] - ad_id = ad[1] - caller.create_ad( - asset, - countrycode, - currency, - provider, - payment_details, - visible=True, - edit=True, - ad_id=ad_id, - ) - - for ad in not_supported_ads: - asset = ad[0] - countrycode = ad[2] - currency = ad[3] - provider = ad[4] - ad_id = ad[1] - caller.create_ad( - asset, - countrycode, - currency, - provider, - payment_details=False, - visible=False, - edit=True, - ad_id=ad_id, - ) - - def distribute_account_details(self, currencies=None, account_info=None): - """ - Helper to distribute the account details for all platforms. - """ - platforms = ("agora", "lbtc") - for platform in platforms: - self._distribute_account_details(platform, currencies=currencies, account_info=account_info) - - def format_ad(self, asset, currency, payment_details_text): - """ - Format the ad. - """ - ad = settings.Platform.Ad - - # Substitute the currency - ad = ad.replace("$CURRENCY$", currency) - - # Substitute the asset - ad = ad.replace("$ASSET$", asset) - - # Substitute the payment details - ad = ad.replace("$PAYMENT$", payment_details_text) - - # Strip extra tabs - ad = ad.replace("\\t", "\t") - return ad - - def format_payment_details(self, currency, payment_details, real=False): - """ - Format the payment details. - """ - if not payment_details: - return False - if real: - payment = settings.Platform.PaymentDetailsReal - else: - payment = settings.Platform.PaymentDetails - - payment_text = "" - for field, value in payment_details.items(): - formatted_name = field.replace("_", " ") - formatted_name = formatted_name.capitalize() - payment_text += f"* {formatted_name}: **{value}**" - if field != list(payment_details.keys())[-1]: # No trailing newline - payment_text += "\n" - - payment = payment.replace("$PAYMENT$", payment_text) - payment = payment.replace("$CURRENCY$", currency) - - return payment diff --git a/handler/money.py b/handler/money.py deleted file mode 100644 index 2dc5ec0..0000000 --- a/handler/money.py +++ /dev/null @@ -1,148 +0,0 @@ -# Twisted imports -from twisted.internet.defer import inlineCallbacks - -# Other library imports -from pycoingecko import CoinGeckoAPI -from forex_python.converter import CurrencyRates - -# Project imports -from settings import settings -import util - - -class Money(util.Base): - """ - Generic class for handling money-related matters that aren't Revolut or Agora. - """ - - def __init__(self): - """ - Initialise the Money object. - Set the logger. - Initialise the CoinGecko API. - """ - super().__init__() - self.cr = CurrencyRates() - self.cg = CoinGeckoAPI() - - def lookup_rates(self, platform, ads, rates=None): - """ - Lookup the rates for a list of public ads. - """ - if not rates: - rates = self.cg.get_price( - ids=["monero", "bitcoin"], - vs_currencies=self.markets.get_all_currencies(platform), - ) - # Set the price based on the asset - for ad in ads: - if ad[4] == "XMR": - coin = "monero" - elif ad[4] == "BTC": - coin = "bitcoin" # No s here - currency = ad[5] - base_currency_price = rates[coin][currency.lower()] - price = float(ad[2]) - rate = round(price / base_currency_price, 2) - ad.append(rate) - # TODO: sort? - return sorted(ads, key=lambda x: x[2]) - - def get_rates_all(self): - """ - Get all rates that pair with USD. - :return: dictionary of USD/XXX rates - :rtype: dict - """ - rates = self.cr.get_rates("USD") - return rates - - def get_acceptable_margins(self, platform, currency, amount): - """ - Get the minimum and maximum amounts we would accept a trade for. - :param currency: currency code - :param amount: amount - :return: (min, max) - :rtype: tuple - """ - sets = util.get_settings(platform) - rates = self.get_rates_all() - if currency == "USD": - min_amount = amount - float(sets.AcceptableUSDMargin) - max_amount = amount + float(sets.AcceptableUSDMargin) - return (min_amount, max_amount) - amount_usd = amount / rates[currency] - min_usd = amount_usd - float(sets.AcceptableUSDMargin) - max_usd = amount_usd + float(sets.AcceptableUSDMargin) - min_local = min_usd * rates[currency] - max_local = max_usd * rates[currency] - return (min_local, max_local) - - def get_minmax(self, platform, asset, currency): - sets = util.get_settings(platform) - rates = self.get_rates_all() - if currency not in rates and not currency == "USD": - self.log.error(f"Can't create ad without rates: {currency}") - return - if asset == "XMR": - min_usd = float(sets.MinUSDXMR) - max_usd = float(sets.MaxUSDXMR) - elif asset == "BTC": - min_usd = float(sets.MinUSDBTC) - max_usd = float(sets.MaxUSDBTC) - if currency == "USD": - min_amount = min_usd - max_amount = max_usd - else: - min_amount = rates[currency] * min_usd - max_amount = rates[currency] * max_usd - - return (min_amount, max_amount) - - def to_usd(self, amount, currency): - if currency == "USD": - return float(amount) - else: - rates = self.get_rates_all() - return float(amount) / rates[currency] - - def multiple_to_usd(self, currency_map): - """ - Convert multiple curencies to USD while saving API calls. - """ - rates = self.get_rates_all() - cumul = 0 - for currency, amount in currency_map.items(): - if currency == "USD": - cumul += float(amount) - else: - cumul += float(amount) / rates[currency] - return cumul - - # TODO: move to money - @inlineCallbacks - def get_profit(self, trades=False): - """ - Check how much total profit we have made. - :return: profit in USD - :rtype: float - """ - total_usd = yield self.tx.get_total_usd() - if not total_usd: - return False - if trades: - trades_usd = yield self.tx.get_open_trades_usd() - total_usd += trades_usd - - profit = total_usd - float(settings.Money.BaseUSD) - if trades: - cast_es = { - "profit_trades_usd": profit, - } - else: - cast_es = { - "profit_usd": profit, - } - - self.tx.write_to_es("get_profit", cast_es) - return profit diff --git a/handler/transactions.py b/handler/transactions.py deleted file mode 100644 index e3150e3..0000000 --- a/handler/transactions.py +++ /dev/null @@ -1,881 +0,0 @@ -# Twisted/Klein imports -from twisted.internet.task import LoopingCall -from twisted.internet.defer import inlineCallbacks - -# Other library imports -from json import dumps -from random import choices -from string import ascii_uppercase -from elasticsearch import Elasticsearch -from datetime import datetime -import urllib3 -import logging - -# Project imports -from settings import settings -import db -import util - -# TODO: secure ES traffic properly -urllib3.disable_warnings() - -tracer = logging.getLogger("elasticsearch") -tracer.setLevel(logging.CRITICAL) -tracer = logging.getLogger("elastic_transport.transport") -tracer.setLevel(logging.CRITICAL) - - -class Transactions(util.Base): - """ - Handler class for incoming Revolut transactions. - """ - - def __init__(self): - """ - Initialise the Transaction object. - Set the logger. - """ - super().__init__() - if settings.ES.Enabled == "1": - self.es = Elasticsearch( - f"https://{settings.ES.Host}:9200", - verify_certs=False, - basic_auth=(settings.ES.Username, settings.ES.Pass), - # ssl_assert_fingerprint=("6b264fd2fd107d45652d8add1750a8a78f424542e13b056d0548173006260710"), - ca_certs="certs/ca.crt", - ) - - def run_checks_in_thread(self): - """ - Run all the balance checks that output into ES in another thread. - """ - - self.get_total() - self.get_remaining() - self.money.get_profit() - self.money.get_profit(True) - self.get_open_trades_usd() - self.get_total_remaining() - self.get_total_with_trades() - - def setup_loops(self): - """ - Set up the LoopingCalls to get the balance so we have data in ES. - """ - if settings.ES.Enabled == "1": - self.lc_es_checks = LoopingCall(self.run_checks_in_thread) - delay = int(settings.ES.RefreshSec) - self.lc_es_checks.start(delay) - self.agora.es = self.es - - def valid_transaction(self, data): - """ - Determine if a given transaction object is valid. - :param data: a transaction cast - :type data: dict - :return: whether the transaction is valid - :rtype: bool - """ - txid = data["transaction_id"] - if "amount" not in data: - return False - if "currency" not in data: - return False - amount = data["amount"] - if amount <= 0: - self.log.info(f"Ignoring transaction with negative/zero amount: {txid}") - return False - return True - - def extract_reference(self, data): - """ - Extract a reference from the transaction cast. - :param data: a transaction cast - :type data: dict - :return: the extracted reference or not_set - :rtype: str - """ - if "reference" in data: - return data["reference"] - elif "meta" in data: - if "provider_reference" in data["meta"]: - return data["meta"]["provider_reference"] - return "not_set" - - def extract_sender(self, data): - """ - Extract a sender name from the transaction cast. - :param data: a transaction cast - :type data: dict - :return: the sender name or not_set - :rtype: str - """ - if "debtorName" in data: - return data["debtorName"] - elif "meta" in data: - if "debtor_account_name" in data["meta"]: - return data["meta"]["debtor_account_name"] - return "not_set" - - def reference_partial_check(self, reference, txid, currency, amount): - """ - Perform a partial check by intersecting all parts of the split of the - reference against the existing references, and returning a set of the matches. - :param reference: the reference to check - :type reference: str - :return: matching trade ID string - :rtype: str - """ - # Partial reference implementation - # Account for silly people not removing the default string - # Split the reference into parts - ref_split = reference.split(" ") - # Get all existing references - existing_refs = db.get_refs() - # Get all parts of the given reference split that match the existing references - stored_trade_reference = set(existing_refs).intersection(set(ref_split)) - if len(stored_trade_reference) > 1: - self.log.error(f"Multiple references valid for TXID {txid}: {reference}") - self.irc.sendmsg(f"Multiple references valid for TXID {txid}: {reference}") - self.ux.notify.notify_tx_lookup_failed(currency, amount, reference, "MULTIPLE_REFS_MATCH") - return False - if len(stored_trade_reference) == 0: - return None - return stored_trade_reference.pop() - - def can_alt_lookup(self, amount, currency, reference): - amount_usd = self.money.to_usd(amount, currency) - # Amount is reliable here as it is checked by find_trade, so no need for stored_trade["amount"] - if float(amount_usd) > float(settings.Agora.AcceptableAltLookupUSD): - self.log.info("Not checking against amount and currency as amount exceeds MAX") - self.irc.sendmsg(f"Not checking against amount and currency as amount exceeds MAX") - # Close here if the amount exceeds the allowable limit for no reference - self.ux.notify.notify_tx_lookup_failed(currency, amount, reference, "EXCEEDS_MAX") - return False - return True - - def amount_currency_lookup(self, amount, currency, txid, reference): - self.log.info(f"No reference in DB refs for {reference}") - self.irc.sendmsg(f"No reference in DB refs for {reference}") - # Try checking just amount and currency, as some people (usually people buying small amounts) - # are unable to put in a reference properly. - - self.log.info(f"Checking against amount and currency for TXID {txid}") - self.irc.sendmsg(f"Checking against amount and currency for TXID {txid}") - if not self.can_alt_lookup(amount, currency, reference): - return False - stored_trade = self.find_trade(txid, currency, amount) - if not stored_trade: - self.log.info(f"Failed to get reference by amount and currency: {txid} {currency} {amount}") - self.irc.sendmsg(f"Failed to get reference by amount and currency: {txid} {currency} {amount}") - self.ux.notify.notify_tx_lookup_failed(currency, amount, reference, "ALT_LOOKUP_FAILED") - return None - stored_trade["amount"] = float(stored_trade["amount"]) # convert to float - return stored_trade - - def normal_lookup(self, stored_trade_reference, reference, currency, amount): - stored_trade = db.get_ref(stored_trade_reference) - if not stored_trade: - self.log.info(f"No reference in DB for {reference}") - self.irc.sendmsg(f"No reference in DB for {reference}") - self.ux.notify.notify_tx_lookup_failed(currency, amount, reference, "NOREF", stored_trade_reference) - return False - stored_trade["amount"] = float(stored_trade["amount"]) # convert to float - return stored_trade - - def currency_check(self, currency, amount, reference, stored_trade): - if not stored_trade["currency"] == currency: - self.log.info(f"Currency mismatch, Agora: {stored_trade['currency']} / Sink: {currency}") - self.irc.sendmsg(f"Currency mismatch, Agora: {stored_trade['currency']} / Sink: {currency}") - self.ux.notify.notify_tx_lookup_failed( - currency, - amount, - reference, - "CURRENCY_MISMATCH", - stored_trade["id"], - ) - return False - return True - - def alt_amount_check(self, platform, amount, currency, reference, stored_trade): - # If the amount does not match exactly, get the min and max values for our given acceptable margins for trades - min_amount, max_amount = self.money.get_acceptable_margins(platform, currency, stored_trade["amount"]) - self.log.info(f"Amount does not match exactly, trying with margins: min: {min_amount} / max: {max_amount}") - self.irc.sendmsg(f"Amount does not match exactly, trying with margins: min: {min_amount} / max: {max_amount}") - if not min_amount < amount < max_amount: - self.log.info( - "Amount mismatch - not in margins: {stored_trade['amount']} (min: {min_amount} / max: {max_amount}" - ) - self.irc.sendmsg( - f"Amount mismatch - not in margins: {stored_trade['amount']} (min: {min_amount} / max: {max_amount}" - ) - self.ux.notify.notify_tx_lookup_failed( - currency, - amount, - reference, - "AMOUNT_MARGIN_MISMATCH", - stored_trade["id"], - ) - return False - return True - - def add_bank_sender(self, platform, platform_buyer, bank_sender): - """ - Add the bank senders into Redis. - :param platform: name of the platform - freeform - :param platform_buyer: the username of the buyer on the platform - :param bank_sender: the sender name from the bank - """ - key = f"namemap.{platform}.{platform_buyer}" - db.r.sadd(key, bank_sender) - - def get_previous_senders(self, platform, platform_buyer): - """ - Get all the previous bank sender names for the given buyer on the platform. - :param platform: name of the platform - freeform - :param platform_buyer: the username of the buyer on the platform - :return: set of previous buyers - :rtype: set - """ - key = f"namemap.{platform}.{platform_buyer}" - senders = db.r.smembers(key) - if not senders: - return None - senders = util.convert(senders) - return senders - - # def get_name_derivations(self, ) - - def check_valid_sender(self, reference, platform, bank_sender, platform_buyer): - """ - Check that either: - * The platform buyer has never had a recognised transaction before - * The bank sender name matches a previous transaction from the platform buyer - :param reference: the trade reference - :param platform: name of the platform - freeform - :param bank_sender: the sender of the bank transaction - :param platform_buyer: the username of the buyer on the platform - :return: whether the sender is valid - :rtype: bool - """ - senders = self.get_previous_senders(platform, platform_buyer) - if senders is None: # no senders yet, assume it's valid - return True - if platform_buyer in senders: - return True - self.ux.notify.notify_sender_name_mismatch(reference, platform_buyer, bank_sender) - return False - - def check_tx_sender(self, tx, reference): - """ - Check whether the sender of a given transaction is authorised based on the previous - transactions of the username that originated the trade reference. - :param tx: the transaction ID - :param reference: the trade reference - """ - stored_trade = db.get_ref(reference) - if not stored_trade: - return None - stored_tx = db.get_tx(tx) - if not stored_tx: - return None - bank_sender = stored_tx["sender"] - platform_buyer = stored_trade["buyer"] - platform = stored_trade["subclass"] - is_allowed = self.check_valid_sender(reference, platform, bank_sender, platform_buyer) - if is_allowed is True: - return True - return False - - def update_trade_tx(self, reference, txid): - """ - Update a trade to point to a given transaction ID. - Return False if the trade already has a mapped transaction. - """ - existing_tx = db.r.hget(f"trade.{reference}", "tx") - if existing_tx is None: - return None - elif existing_tx == b"": - db.r.hset(f"trade.{reference}", "tx", txid) - return True - else: # Already a mapped transaction - return False - - def transaction(self, data): - """ - Store details of transaction and post notifications to IRC. - Matches it up with data stored in Redis to attempt to reconcile with an Agora trade. - :param data: details of transaction - :type data: dict - """ - valid = self.valid_transaction(data) - if not valid: - return False - ts = data["timestamp"] - txid = data["transaction_id"] - amount = float(data["amount"]) - currency = data["currency"] - - reference = self.extract_reference(data) - sender = self.extract_sender(data) - - subclass = data["subclass"] - to_store = { - "subclass": subclass, - "ts": ts, - "txid": txid, - "reference": reference, - "amount": amount, - "currency": currency, - "sender": sender, - } - db.r.hmset(f"tx.{txid}", to_store) - - self.log.info(f"Transaction processed: {dumps(to_store, indent=2)}") - self.irc.sendmsg(f"AUTO Incoming transaction on {subclass}: {txid} {amount}{currency} ({reference})") - - stored_trade_reference = self.reference_partial_check(reference, txid, currency, amount) - if stored_trade_reference is False: # can be None though - return - - stored_trade = False - looked_up_without_reference = False - - # Normal implementation for when we have a reference - if stored_trade_reference: - stored_trade = self.normal_lookup(stored_trade_reference, reference, currency, amount) - # if not stored_trade: - # return - - # Amount/currency lookup implementation for when we have no reference - else: - if not stored_trade: # check we don't overwrite the lookup above - stored_trade = self.amount_currency_lookup(amount, currency, txid, reference) - if stored_trade is False: - return - if stored_trade: - # Note that we have looked it up without reference so we don't use +- below - # This might be redundant given the amount checks in find_trade, but better safe than sorry! - looked_up_without_reference = True - else: - return - else: - # Stored trade reference is none, the checks below will do nothing at all - return - - # Make sure it was sent in the expected currency - if not self.currency_check(currency, amount, reference, stored_trade): - return - - # Make sure the expected amount was sent - if not stored_trade["amount"] == amount: - if looked_up_without_reference: - return - platform = stored_trade["subclass"] - if not self.alt_amount_check(platform, amount, currency, reference, stored_trade): - return - platform = stored_trade["subclass"] - platform_buyer = stored_trade["buyer"] - - # Check sender - we don't do anything with this yet - sender_valid = self.check_valid_sender(reference, platform, sender, platform_buyer) - self.log.info(f"Trade {reference} buyer {platform_buyer} valid: {sender_valid}") - # trade_released = self.release_map_trade(reference, txid) - # if trade_released: - # self.ux.notify.notify_complete_trade(amount, currency) - # else: - # self.log.error(f"Cannot release trade {reference}.") - # return - - self.release_funds(stored_trade["id"], stored_trade["reference"]) - - def release_funds(self, trade_id, reference): - stored_trade = db.get_ref(reference) - platform = stored_trade["subclass"] - logmessage = f"All checks passed, releasing funds for {trade_id} {reference}" - self.log.info(logmessage) - self.irc.sendmsg(logmessage) - if platform == "agora": - release = self.agora.release_funds - post_message = self.agora.agora.contact_message_post - elif platform == "lbtc": - release = self.lbtc.release_funds - post_message = self.lbtc.lbtc.contact_message_post - - rtrn = release(trade_id) - if rtrn["message"] == "OK": - post_message(trade_id, "Thanks! Releasing now :)") - else: - logmessage = f"Release funds unsuccessful: {rtrn['message']}" - self.log.error(logmessage) - self.irc.sendmsg(logmessage) - self.ux.notify.notify_release_unsuccessful(trade_id) - return - - # Parse the escrow release response - message = rtrn["message"] - # message_long = rtrn["response"]["data"]["message"] - self.irc.sendmsg(f"{dumps(message)}") - - def release_map_trade(self, reference, tx): - """ - Map a trade to a transaction and release if no other TX is - mapped to the same trade. - """ - stored_trade = db.get_ref(reference) - if not stored_trade: - self.log.error(f"Could not get stored trade for {reference}.") - return None - tx_obj = db.get_tx(tx) - if not tx_obj: - self.log.error(f"Could not get TX for {tx}.") - return None - platform = stored_trade["subclass"] - platform_buyer = stored_trade["buyer"] - bank_sender = tx_obj["sender"] - trade_id = stored_trade["id"] - is_updated = self.update_trade_tx(reference, tx) - if is_updated is None: - return None - elif is_updated is True: - # We mapped the trade successfully - self.release_funds(trade_id, reference) - self.add_bank_sender(platform, platform_buyer, bank_sender) - return True - elif is_updated is False: - # Already mapped - self.log.error(f"Trade {reference} already has a TX mapped, cannot map {tx}.") - return False - - def find_trades_by_uid(self, uid): - """ - Find a list of trade IDs and references by a customer UID. - :return: tuple of (platform, trade_id, reference, currency) - """ - platform, username = self.get_uid(uid) - refs = db.get_refs() - matching_trades = [] - for reference in refs: - ref_data = db.get_ref(reference) - tx_platform = ref_data["subclass"] - tx_username = ref_data["buyer"] - trade_id = ref_data["id"] - currency = ref_data["currency"] - if tx_platform == platform and tx_username == username: - to_append = (platform, trade_id, reference, currency) - matching_trades.append(to_append) - return matching_trades - - def user_verification_successful(self, uid): - """ - A user has successfully completed verification. - """ - self.log.info(f"User has completed verification: {uid}") - trade_list = self.find_trades_by_uid(uid) - for platform, trade_id, reference, currency in trade_list: - self.send_bank_details(platform, currency, trade_id) - self.send_reference(platform, trade_id, reference) - - def create_uid(self, platform, username): - return f"{platform}|{username}" - - def get_uid(self, external_user_id): - """ - Get the platform and username from the external user ID. - """ - spl = external_user_id.split("|") - if not len(spl) == 2: - self.log.error(f"Split invalid, cannot get customer: {spl}") - return False - platform, username = spl - return (platform, username) - - def get_send_settings(self, platform): - if platform == "agora": - send_setting = settings.Agora.Send - post_message = self.agora.agora.contact_message_post - elif platform == "lbtc": - send_setting = settings.LocalBitcoins.Send - post_message = self.lbtc.lbtc.contact_message_post - - return (send_setting, post_message) - - def send_reference(self, platform, trade_id, reference): - """ - Send the reference to a customer. - """ - send_setting, post_message = self.get_send_settings(platform) - if send_setting == "1": - post_message( - trade_id, - f"When sending the payment please use reference code: {reference}", - ) - - def send_verification_url(self, platform, uid, trade_id): - send_setting, post_message = self.get_send_settings(platform) - if send_setting == "1": - auth_url = self.ux.verify.create_applicant_and_get_link(uid) - if platform == "lbtc": - auth_url = auth_url.replace("https://", "") # hack - post_message( - trade_id, - f"Hi! To continue the trade, please complete the verification form: {auth_url}", - ) - - def send_bank_details(self, platform, currency, trade_id): - """ - Send the bank details to a trade. - """ - send_setting, post_message = self.get_send_settings(platform) - self.log.info(f"Sending bank details/reference for {platform}/{trade_id}") - if send_setting == "1": - account_info = self.markets.get_matching_account_details(platform, currency) - formatted_account_info = self.markets.format_payment_details(currency, account_info, real=True) - post_message( - trade_id, - f"Payment details: \n{formatted_account_info}", - ) - - def new_trade( - self, - subclass, - asset, - trade_id, - buyer, - currency, - amount, - amount_crypto, - provider, - ): - """ - Called when we have a new trade in Agora. - Store details in Redis, generate a reference and optionally let the customer know the reference. - """ - reference = "".join(choices(ascii_uppercase, k=5)) - reference = f"PGN-{reference}" - existing_ref = db.r.get(f"trade.{trade_id}.reference") - if not existing_ref: - to_store = { - "id": trade_id, - "tx": "", - "asset": asset, - "buyer": buyer, - "currency": currency, - "amount": amount, - "amount_crypto": amount_crypto, - "reference": reference, - "provider": provider, - "subclass": subclass, - } - self.log.info(f"Storing trade information: {str(to_store)}") - db.r.hmset(f"trade.{reference}", to_store) - db.r.set(f"trade.{trade_id}.reference", reference) - self.irc.sendmsg(f"Generated reference for {trade_id}: {reference}") - self.ux.notify.notify_new_trade(amount, currency) - uid = self.create_uid(subclass, buyer) - verified = self.ux.verify.get_external_user_id_status(uid) - if verified != "GREEN": - self.log.info(f"UID {uid} is not verified, sending link.") - self.send_verification_url(subclass, uid, trade_id) - else: # User is verified - self.log.info(f"UID {uid} is verified.") - self.send_bank_details(subclass, currency, trade_id) - self.send_reference(subclass, trade_id, reference) - if existing_ref: - return util.convert(existing_ref) - else: - return reference - - def find_trade(self, txid, currency, amount): - """ - Get a trade reference that matches the given currency and amount. - Only works if there is one result. - :param txid: Sink transaction ID - :param currency: currency - :param amount: amount - :type txid: string - :type currency: string - :type amount: int - :return: matching trade object or False - :rtype: dict or bool - """ - refs = db.get_refs() - matching_refs = [] - # TODO: use get_ref_map in this function instead of calling get_ref multiple times - for ref in refs: - stored_trade = db.get_ref(ref) - if stored_trade["currency"] == currency and float(stored_trade["amount"]) == float(amount): - matching_refs.append(stored_trade) - if len(matching_refs) != 1: - self.log.error(f"Find trade returned multiple results for TXID {txid}: {matching_refs}") - return False - return matching_refs[0] - - @inlineCallbacks - def get_total_usd(self): - """ - Get total USD in all our accounts, bank and trading. - :return: value in USD - :rtype float: - """ - total_sinks_usd = self.sinks.get_total_usd() - agora_wallet_xmr = yield self.agora.agora.wallet_balance_xmr() - agora_wallet_btc = yield self.agora.agora.wallet_balance() - lbtc_wallet_btc = yield self.lbtc.lbtc.wallet_balance() - if not agora_wallet_xmr["success"]: - return False - if not agora_wallet_btc["success"]: - return False - if not lbtc_wallet_btc["success"]: - return False - if not agora_wallet_xmr["response"]: - return False - if not agora_wallet_btc["response"]: - return False - if not lbtc_wallet_btc["response"]: - return False - total_xmr_agora = agora_wallet_xmr["response"]["data"]["total"]["balance"] - total_btc_agora = agora_wallet_btc["response"]["data"]["total"]["balance"] - total_btc_lbtc = lbtc_wallet_btc["response"]["data"]["total"]["balance"] - # Get the XMR -> USD exchange rate - xmr_usd = self.money.cg.get_price(ids="monero", vs_currencies=["USD"]) - - # Get the BTC -> USD exchange rate - btc_usd = self.money.cg.get_price(ids="bitcoin", vs_currencies=["USD"]) - - # Convert the Agora BTC total to USD - total_usd_agora_btc = float(total_btc_agora) * btc_usd["bitcoin"]["usd"] - - # Convert the LBTC BTC total to USD - total_usd_lbtc_btc = float(total_btc_lbtc) * btc_usd["bitcoin"]["usd"] - - # Convert the Agora XMR total to USD - total_usd_agora_xmr = float(total_xmr_agora) * xmr_usd["monero"]["usd"] - - # Add it all up - total_usd_agora = total_usd_agora_xmr + total_usd_agora_btc - total_usd_lbtc = total_usd_lbtc_btc - total_usd = total_usd_agora + total_usd_lbtc + total_sinks_usd - cast_es = { - "price_usd": total_usd, - "total_usd_agora_xmr": total_usd_agora_xmr, - "total_usd_agora_btc": total_usd_agora_btc, - "total_usd_lbtc_btc": total_usd_lbtc_btc, - "total_xmr_agora": total_xmr_agora, - "total_btc_agora": total_btc_agora, - "total_btc_lbtc": total_btc_lbtc, - "xmr_usd": xmr_usd["monero"]["usd"], - "btc_usd": btc_usd["bitcoin"]["usd"], - "total_sinks_usd": total_sinks_usd, - "total_usd_agora": total_usd_agora, - } - self.write_to_es("get_total_usd", cast_es) - return total_usd - - # TODO: possibly refactor this into smaller functions which don't return as much stuff - # check if this is all really needed in the corresponding withdraw function - @inlineCallbacks - def get_total(self): - """ - Get all the values corresponding to the amount of money we hold. - :return: ((total SEK, total USD, total GBP), (total XMR USD, total BTC USD), (total XMR, total BTC)) - :rtype: tuple(tuple(float, float, float), tuple(float, float), tuple(float, float)) - """ - total_sinks_usd = self.sinks.get_total_usd() - agora_wallet_xmr = yield self.agora.agora.wallet_balance_xmr() - if not agora_wallet_xmr["success"]: - self.log.error("Could not get Agora XMR wallet total.") - return False - agora_wallet_btc = yield self.agora.agora.wallet_balance() - if not agora_wallet_btc["success"]: - self.log.error("Could not get Agora BTC wallet total.") - return False - lbtc_wallet_btc = yield self.lbtc.lbtc.wallet_balance() - if not lbtc_wallet_btc["success"]: - return False - total_xmr_agora = agora_wallet_xmr["response"]["data"]["total"]["balance"] - total_btc_agora = agora_wallet_btc["response"]["data"]["total"]["balance"] - total_btc_lbtc = lbtc_wallet_btc["response"]["data"]["total"]["balance"] - # Get the XMR -> USD exchange rate - xmr_usd = self.money.cg.get_price(ids="monero", vs_currencies=["USD"]) - - # Get the BTC -> USD exchange rate - btc_usd = self.money.cg.get_price(ids="bitcoin", vs_currencies=["USD"]) - - # Convert the Agora XMR total to USD - total_usd_agora_xmr = float(total_xmr_agora) * xmr_usd["monero"]["usd"] - - # Convert the Agora BTC total to USD - total_usd_agora_btc = float(total_btc_agora) * btc_usd["bitcoin"]["usd"] - - # Convert the LBTC BTC total to USD - total_usd_lbtc_btc = float(total_btc_lbtc) * btc_usd["bitcoin"]["usd"] - - # Add it all up - total_usd_agora = total_usd_agora_xmr + total_usd_agora_btc - total_usd_lbtc = total_usd_lbtc_btc - total_usd = total_usd_agora + total_usd_lbtc + total_sinks_usd - - total_btc_usd = total_usd_agora_btc + total_usd_lbtc_btc - total_xmr_usd = total_usd_agora_xmr - - total_xmr = total_xmr_agora - total_btc = total_btc_lbtc + total_btc_agora - - # Convert the total USD price to GBP and SEK - rates = self.money.get_rates_all() - price_sek = rates["SEK"] * total_usd - price_usd = total_usd - price_gbp = rates["GBP"] * total_usd - - cast = ( - ( - price_sek, - price_usd, - price_gbp, - ), # Total prices in our 3 favourite currencies - ( - total_xmr_usd, - total_btc_usd, - ), # Total USD balance in only Agora - (total_xmr, total_btc), - ) # Total XMR and BTC balance in Agora - - cast_es = { - "price_sek": price_sek, - "price_usd": price_usd, - "price_gbp": price_gbp, - "total_usd_agora_xmr": total_usd_agora_xmr, - "total_usd_agora_btc": total_usd_agora_btc, - "total_usd_lbtc_btc": total_usd_lbtc_btc, - "total_xmr_agora": total_xmr_agora, - "total_btc_agora": total_btc_agora, - "total_btc_lbtc": total_btc_lbtc, - "xmr_usd": xmr_usd["monero"]["usd"], - "btc_usd": btc_usd["bitcoin"]["usd"], - "total_sinks_usd": total_sinks_usd, - "total_usd_agora": total_usd_agora, - } - self.write_to_es("get_total", cast_es) - return cast - - def write_to_es(self, msgtype, cast): - if settings.ES.Enabled == "1": - cast["type"] = msgtype - cast["ts"] = str(datetime.now().isoformat()) - cast["xtype"] = "tx" - self.es.index(index=settings.ES.Index, document=cast) - - @inlineCallbacks - def get_remaining(self): - """ - Check how much profit we need to make in order to withdraw. - :return: profit remaining in USD - :rtype: float - """ - total_usd = yield self.get_total_usd() - if not total_usd: - return False - - withdraw_threshold = float(settings.Money.BaseUSD) + float(settings.Money.WithdrawLimit) - remaining = withdraw_threshold - total_usd - cast_es = { - "remaining_usd": remaining, - } - self.write_to_es("get_remaining", cast_es) - return remaining - - def open_trades_usd_parse_dash(self, dash, rates): - cumul_usd = 0 - for contact_id, contact in dash.items(): - # We need created at in order to look up the historical prices - created_at = contact["data"]["created_at"] - - # Reformat the date how CoinGecko likes - # 2022-05-02T11:17:14+00:00 - date_parsed = datetime.strptime(created_at, "%Y-%m-%dT%H:%M:%S.%fZ") - date_formatted = date_parsed.strftime("%d-%m-%Y") - - # Get the historical rates for the right asset, extract the price - asset = contact["data"]["advertisement"]["asset"] - if asset == "XMR": - amount_crypto = contact["data"]["amount_xmr"] - history = self.money.cg.get_coin_history_by_id(id="monero", date=date_formatted) - crypto_usd = float(history["market_data"]["current_price"]["usd"]) - elif asset == "BTC": - amount_crypto = contact["data"]["amount_btc"] - history = self.money.cg.get_coin_history_by_id(id="bitcoin", date=date_formatted) - crypto_usd = float(history["market_data"]["current_price"]["usd"]) - # Convert crypto to fiat - amount = float(amount_crypto) * crypto_usd - currency = contact["data"]["currency"] - if not contact["data"]["is_selling"]: - continue - if currency == "USD": - cumul_usd += float(amount) - else: - rate = rates[currency] - amount_usd = float(amount) / rate - cumul_usd += amount_usd - return cumul_usd - - @inlineCallbacks - def get_open_trades_usd(self): - """ - Get total value of open trades in USD. - :return: total trade value - :rtype: float - """ - dash_agora = self.agora.wrap_dashboard() - dash_lbtc = self.lbtc.wrap_dashboard() - dash_agora = yield dash_agora - dash_lbtc = yield dash_lbtc - if dash_agora is False: - return False - if dash_lbtc is False: - return False - - rates = self.money.get_rates_all() - cumul_usd_agora = self.open_trades_usd_parse_dash(dash_agora, rates) - cumul_usd_lbtc = self.open_trades_usd_parse_dash(dash_lbtc, rates) - cumul_usd = cumul_usd_agora + cumul_usd_lbtc - - cast_es = { - "trades_usd": cumul_usd, - } - self.write_to_es("get_open_trades_usd", cast_es) - return cumul_usd - - @inlineCallbacks - def get_total_remaining(self): - """ - Check how much profit we need to make in order to withdraw, taking into account open trade value. - :return: profit remaining in USD - :rtype: float - """ - total_usd = yield self.get_total_usd() - total_trades_usd = yield self.get_open_trades_usd() - if not total_usd: - return False - total_usd += total_trades_usd - withdraw_threshold = float(settings.Money.BaseUSD) + float(settings.Money.WithdrawLimit) - remaining = withdraw_threshold - total_usd - - cast_es = { - "total_remaining_usd": remaining, - } - self.write_to_es("get_total_remaining", cast_es) - return remaining - - @inlineCallbacks - def get_total_with_trades(self): - total_usd = yield self.get_total_usd() - if not total_usd: - return False - total_trades_usd = yield self.get_open_trades_usd() - total_with_trades = total_usd + total_trades_usd - cast_es = { - "total_with_trades": total_with_trades, - } - self.write_to_es("get_total_with_trades", cast_es) - return total_with_trades