From 6504c440e09ae2a49be1c31c42eb857f44600bdf Mon Sep 17 00:00:00 2001 From: Mark Veidemanis Date: Thu, 5 May 2022 12:50:41 +0100 Subject: [PATCH] Move common platform code into a Local library --- handler/lib/agoradesk_py.py | 3 + handler/lib/localbitcoins_py.py | 5 +- handler/sources/agora.py | 615 +---------------------------- handler/sources/local.py | 658 +++++++++++++++++++++++++++++++ handler/sources/localbitcoins.py | 611 +--------------------------- handler/tests/test_agora.py | 10 +- handler/tests/test_lbtc.py | 10 +- 7 files changed, 689 insertions(+), 1223 deletions(-) diff --git a/handler/lib/agoradesk_py.py b/handler/lib/agoradesk_py.py index 0ac0168..7170a7f 100644 --- a/handler/lib/agoradesk_py.py +++ b/handler/lib/agoradesk_py.py @@ -564,6 +564,7 @@ class AgoraDesk: visible: Optional[bool] = None, asset: Optional[str] = None, payment_method_code: Optional[str] = None, + page: Optional[int] = None, ) -> Dict[str, Any]: """See Agoradesk API. @@ -588,6 +589,8 @@ class AgoraDesk: params["asset"] = asset if payment_method_code: params["payment_method_code"] = payment_method_code + if page: + params["page"] = page if len(params) == 0: return self._api_call(api_method="ads") diff --git a/handler/lib/localbitcoins_py.py b/handler/lib/localbitcoins_py.py index 26603b3..649fa4f 100644 --- a/handler/lib/localbitcoins_py.py +++ b/handler/lib/localbitcoins_py.py @@ -614,6 +614,7 @@ class LocalBitcoins: visible: Optional[bool] = None, asset: Optional[str] = None, payment_method_code: Optional[str] = None, + page: Optional[int] = None, ) -> Dict[str, Any]: """See LocalBitcoins API. @@ -638,6 +639,8 @@ class LocalBitcoins: params["asset"] = asset if payment_method_code: params["payment_method_code"] = payment_method_code + if page: + params["page"] = page if len(params) == 0: return self._api_call(api_method="api/ads/") @@ -724,7 +727,7 @@ class LocalBitcoins: params = self._generic_search_parameters(amount, page) return self._api_call( - api_method=f"{direction}-{main_currency}-online/" f"{exchange_currency}{add_to_api_method}", + api_method=f"{direction}-{main_currency}-online/" f"{exchange_currency}{add_to_api_method}/.json", query_values=params, ) diff --git a/handler/sources/agora.py b/handler/sources/agora.py index 6ac38aa..ec0483d 100644 --- a/handler/sources/agora.py +++ b/handler/sources/agora.py @@ -1,22 +1,13 @@ -# Twisted/Klein imports -from twisted.internet.task import LoopingCall -from twisted.internet.defer import inlineCallbacks - # Other library imports -from json import loads -from lib.agoradesk_py import AgoraDesk -from time import sleep from pyotp import TOTP -from datetime import datetime # Project imports from settings import settings import util - -# import sources.local +import sources.local -class Agora(util.Base): +class Agora(sources.local.Local): """ AgoraDesk API handler. """ @@ -28,7 +19,6 @@ class Agora(util.Base): """ self.platform = "agora" super().__init__() - self.agora = AgoraDesk(settings.Agora.Token) # Cache for detecting new trades self.last_dash = set() @@ -39,597 +29,6 @@ class Agora(util.Base): # Assets that cheat has been run on self.cheat_run_on = [] - def setup_loop(self): # TODO:: move to main sources - """ - Set up the LoopingCall to get all active trades and messages. - """ - self.log.debug("Setting up loops.") - self.lc_dash = LoopingCall(self.loop_check) - self.lc_dash.start(int(settings.Agora.RefreshSec)) - if settings.Agora.Cheat == "1": - self.lc_cheat = LoopingCall(self.run_cheat_in_thread) - self.lc_cheat.start(int(settings.Agora.CheatSec)) - self.log.debug("Finished setting up loops.") - - @inlineCallbacks - def got_dashboard(self, dash): - dash_tmp = yield self.wrap_dashboard(dash) - self.dashboard_hook(dash_tmp) - - @inlineCallbacks - def wrap_dashboard(self, dash=None): # backwards compatibility with TX - if not dash: - dash = yield self.agora.dashboard_seller() - # if dash["response"] is None: - # return False - dash_tmp = {} - if not dash: - return False - if not dash["response"]: - return False - if "data" not in dash["response"]: - # self.log.error(f"Data not in dashboard response: {dash}") - return dash_tmp - if dash["response"]["data"]["contact_count"] > 0: - for contact in dash["response"]["data"]["contact_list"]: - contact_id = contact["data"]["contact_id"] - dash_tmp[contact_id] = contact - return dash_tmp - - def loop_check(self): - """ - Calls hooks to parse dashboard info and get all contact messages. - """ - d = self.agora.dashboard_seller() - d.addCallback(self.got_dashboard) - - # Get recent messages - m = self.agora.recent_messages() - m.addCallback(self.got_recent_messages) - - @inlineCallbacks - def get_dashboard_irc(self): - """ - Get dashboard helper for IRC only. - """ - dash = yield self.wrap_dashboard() - rtrn = [] - if dash is False: - return False - for contact_id, contact in dash.items(): - reference = self.tx.tx_to_ref(contact_id) - buyer = contact["data"]["buyer"]["username"] - amount = contact["data"]["amount"] - asset = contact["data"]["advertisement"]["asset"] - if asset == "XMR": - amount_crypto = contact["data"]["amount_xmr"] - elif asset == "BTC": - amount_crypto = contact["data"]["amount_btc"] - currency = contact["data"]["currency"] - provider = contact["data"]["advertisement"]["payment_method"] - if not contact["data"]["is_selling"]: - continue - rtrn.append( - ( - f"[#] [{reference}] ({self.platform}) <{buyer}>" - f" {amount}{currency} {provider} {amount_crypto}{asset}" - ) - ) - return rtrn - - def dashboard_hook(self, dash): - """ - Get information about our open trades. - Post new trades to IRC and cache trades for the future. - """ - current_trades = [] - if not dash: - return - if not dash.items(): - return - for contact_id, contact in dash.items(): - reference = self.tx.tx_to_ref(contact_id) - if reference: - current_trades.append(reference) - buyer = contact["data"]["buyer"]["username"] - amount = contact["data"]["amount"] - asset = contact["data"]["advertisement"]["asset"] - provider = contact["data"]["advertisement"]["payment_method"] - if asset == "XMR": - amount_crypto = contact["data"]["amount_xmr"] - elif asset == "BTC": - amount_crypto = contact["data"]["amount_btc"] - currency = contact["data"]["currency"] - if not contact["data"]["is_selling"]: - continue - if reference not in self.last_dash: - reference = self.tx.new_trade( - self.platform, - asset, - contact_id, - buyer, - currency, - amount, - amount_crypto, - provider, - ) - if reference: - if reference not in current_trades: - current_trades.append(reference) - # Let us know there is a new trade - self.irc.sendmsg( - ( - f"[#] [{reference}] ({self.platform}) <{buyer}>" - f" {amount}{currency} {provider} {amount_crypto}{asset}" - ) - ) - # Note that we have seen this reference - self.last_dash.add(reference) - - # Purge old trades from cache - for ref in list(self.last_dash): # We're removing from the list on the fly - if ref not in current_trades: - self.last_dash.remove(ref) - if reference and reference not in current_trades: - current_trades.append(reference) - self.tx.cleanup(self.platform, current_trades) - - def got_recent_messages(self, messages, send_irc=True): - """ - Get recent messages. - """ - messages_tmp = {} - if not messages: - return False - if not messages["success"]: - return False - if "data" not in messages["response"]: - self.log.error(f"Data not in messages response: {messages['response']}") - return False - open_tx = self.tx.get_ref_map().keys() - for message in messages["response"]["data"]["message_list"]: - contact_id = message["contact_id"] - username = message["sender"]["username"] - msg = message["msg"] - if contact_id not in open_tx: - continue - reference = self.tx.tx_to_ref(contact_id) - if reference in messages_tmp: - messages_tmp[reference].append([username, msg]) - else: - messages_tmp[reference] = [[username, msg]] - - # Send new messages on IRC - if send_irc: - for user, message in messages_tmp[reference][::-1]: - if reference in self.last_messages: - if not [user, message] in self.last_messages[reference]: - self.irc.sendmsg(f"[{reference}] ({self.platform}) <{user}> {message}") - # Append sent messages to last_messages so we don't send them again - self.last_messages[reference].append([user, message]) - else: - self.last_messages[reference] = [[user, message]] - for x in messages_tmp[reference]: - self.irc.sendmsg(f"[{reference}] ({self.platform}) <{user}> {message}") - - # Purge old trades from cache - for ref in list(self.last_messages): # We're removing from the list on the fly - if ref not in messages_tmp: - del self.last_messages[ref] - - return messages_tmp - - @inlineCallbacks - def enum_ad_ids(self, page=0): - ads = yield self.agora._api_call(api_method="ads", query_values={"page": page}) - if ads is False: - return False - ads_total = [] - if not ads["success"]: - return False - for ad in ads["response"]["data"]["ad_list"]: - ads_total.append(ad["data"]["ad_id"]) - if "pagination" in ads["response"]: - if "next" in ads["response"]["pagination"]: - page += 1 - ads_iter = yield self.enum_ad_ids(page) - if ads_iter is None: - return False - if ads_iter is False: - return False - for ad in ads_iter: - ads_total.append(ad) - return ads_total - - @inlineCallbacks - def enum_ads(self, requested_asset=None, page=0): - query_values = {"page": page} - if requested_asset: - query_values["asset"] = requested_asset - ads = yield self.agora._api_call(api_method="ads", query_values=query_values) - if ads is False: - return False - ads_total = [] - if not ads["success"]: - return False - for ad in ads["response"]["data"]["ad_list"]: - asset = ad["data"]["asset"] - ad_id = ad["data"]["ad_id"] - country = ad["data"]["countrycode"] - currency = ad["data"]["currency"] - provider = ad["data"]["online_provider"] - ads_total.append([asset, ad_id, country, currency, provider]) - if "pagination" in ads["response"]: - if "next" in ads["response"]["pagination"]: - page += 1 - ads_iter = yield self.enum_ads(requested_asset, page) - if ads_iter is None: - return False - if ads_iter is False: - return False - for ad in ads_iter: - ads_total.append([ad[0], ad[1], ad[2], ad[3], ad[4]]) - return ads_total - - @inlineCallbacks - def enum_public_ads(self, asset, currency, providers=None, page=0): - to_return = [] - if asset == "XMR": - coin = "monero" - elif asset == "BTC": - coin = "bitcoins" - if not providers: - providers = ["REVOLUT"] - # buy-monero-online, buy-bitcoin-online - # Work around Agora weirdness calling it bitcoins - ads = yield self.agora._api_call( - api_method=f"buy-{coin}-online/{currency}", - query_values={"page": page}, - ) - # with open("pub.json", "a") as f: - # import json - # f.write(json.dumps([page, currency, asset, ads])+"\n") - # f.close() - if ads is None: - return False - if ads is False: - return False - if ads["response"] is None: - return False - if "data" not in ads["response"]: - return False - for ad in ads["response"]["data"]["ad_list"]: - if ad["data"]["online_provider"] not in providers: - continue - date_last_seen = ad["data"]["profile"]["last_online"] - # Check if this person was seen recently - if not util.last_online_recent(date_last_seen): - continue - ad_id = ad["data"]["ad_id"] - username = ad["data"]["profile"]["username"] - temp_price = ad["data"]["temp_price"] - provider = ad["data"]["online_provider"] - if ad["data"]["currency"] != currency: - continue - to_append = [ad_id, username, temp_price, provider, asset, currency] - if to_append not in to_return: - to_return.append(to_append) - # yield [ad_id, username, temp_price, provider, asset, currency] - if "pagination" in ads["response"]: - if "next" in ads["response"]["pagination"]: - page += 1 - ads_iter = yield self.enum_public_ads(asset, currency, providers, page) - if ads_iter is None: - return False - if ads_iter is False: - return False - for ad in ads_iter: - to_append = [ad[0], ad[1], ad[2], ad[3], ad[4], ad[5]] - if to_append not in to_return: - to_return.append(to_append) - return to_return - - def run_cheat_in_thread(self, assets=None): - """ - Update prices in another thread. - """ - if not assets: - all_assets = loads(settings.Agora.AssetList) - assets_not_run = set(all_assets) ^ set(self.cheat_run_on) - if not assets_not_run: - self.cheat_run_on = [] - asset = list(all_assets).pop() - self.cheat_run_on.append(asset) - else: - asset = assets_not_run.pop() - self.cheat_run_on.append(asset) - self.update_prices([asset]) - return asset - else: - # deferToThread(self.update_prices, assets) - self.update_prices(assets) - - @inlineCallbacks - def update_prices(self, assets=None): - # Get all public ads for the given assets - public_ads = yield self.get_all_public_ads(assets) - if not public_ads: - return False - - # Get the ads to update - to_update = self.markets.get_new_ad_equations(self.platform, public_ads, assets) - self.slow_ad_update(to_update) - - # TODO: make generic and move to markets - @inlineCallbacks - def get_all_public_ads(self, assets=None, currencies=None, providers=None): - """ - Get all public ads for our listed currencies. - :return: dict of public ads keyed by currency - :rtype: dict - """ - public_ads = {} - crypto_map = { - "XMR": "monero", - "BTC": "bitcoin", - } - - if not assets: - assets = self.markets.get_all_assets(self.platform) - # Get all currencies we have ads for, deduplicated - if not currencies: - currencies = self.markets.get_all_currencies(self.platform) - if not providers: - providers = self.markets.get_all_providers(self.platform) - sinks_currencies = self.sinks.currencies - supported_currencies = [currency for currency in currencies if currency in sinks_currencies] - currencies = supported_currencies - # We want to get the ads for each of these currencies and return the result - rates = self.money.cg.get_price(ids=["monero", "bitcoin"], vs_currencies=currencies) - for asset in assets: - for currency in currencies: - cg_asset_name = crypto_map[asset] - try: - rates[cg_asset_name][currency.lower()] - except KeyError: - # self.log.error("Error getting public ads for currency {currency}", currency=currency) - continue - ads_list = yield self.enum_public_ads(asset, currency, providers) - if not ads_list: - continue - ads = self.money.lookup_rates(self.platform, ads_list, rates=rates) - if not ads: - continue - self.write_to_es_ads("ads", ads) - if currency in public_ads: - for ad in list(ads): - if ad not in public_ads[currency]: - public_ads[currency].append(ad) - else: - public_ads[currency] = ads - - return public_ads - - def write_to_es_ads(self, msgtype, ads): - if settings.ES.Enabled == "1": - for ad in ads: - cast = { - "id": ad[0], - "username": ad[1], - "price": ad[2], - "provider": ad[3], - "asset": ad[4], - "currency": ad[5], - "margin": ad[6], - } - cast["type"] = msgtype - cast["ts"] = str(datetime.now().isoformat()) - cast["xtype"] = "platorm" - cast["market"] = self.platform - self.es.index(index=settings.ES.MetaIndex, document=cast) - - @inlineCallbacks - def slow_ad_update(self, ads): - """ - Slow ad equation update utilising exponential backoff in order to guarantee all ads are updated. - :param ads: our list of ads - """ - iterations = 0 - throttled = 0 - assets = set() - currencies = set() - while not all([x[4] for x in ads]) or iterations == 1000: - for ad_index in range(len(ads)): - ad_id, new_formula, asset, currency, actioned = ads[ad_index] - assets.add(asset) - currencies.add(currency) - if not actioned: - rtrn = yield self.agora.ad_equation(ad_id, new_formula) - if rtrn["success"]: - ads[ad_index][4] = True - throttled = 0 - continue - else: - if "error_code" not in rtrn["response"]["error"]: - self.log.error(f"Error code not in return for ad {ad_id}: {rtrn['response']}") - return - if rtrn["response"]["error"]["error_code"] == 429: - throttled += 1 - sleep_time = pow(throttled, float(settings.Agora.SleepExponent)) - self.log.info( - f"Throttled {throttled} times while updating {ad_id}, sleeping for {sleep_time} seconds" - ) - # We're running in a thread, so this is fine - sleep(sleep_time) - self.log.error(f"Error updating ad {ad_id}: {rtrn['response']}") - continue - iterations += 1 - - @util.handle_exceptions - def nuke_ads(self): - """ - Delete all of our adverts. - :return: True or False - :rtype: bool - """ - ads = self.enum_ad_ids() - return_ids = [] - if ads is False: - return False - for ad_id in ads: - rtrn = self.agora.ad_delete(ad_id) - return_ids.append(rtrn["success"]) - return all(return_ids) - - @util.handle_exceptions - def create_ad( - self, - asset, - countrycode, - currency, - provider, - payment_details, - visible=None, - edit=False, - ad_id=None, - ): - """ - Post an ad with the given asset in a country with a given currency. - Convert the min and max amounts from settings to the given currency with CurrencyRates. - :param asset: the crypto asset to list (XMR or BTC) - :type asset: string - :param countrycode: country code - :param currency: currency code - :param payment_details: the payment details - :type countrycode: string - :type currency: string - :type payment_details: dict - :return: data about created object or error - :rtype: dict - """ - - if payment_details: - payment_details_text = self.markets.format_payment_details(currency, payment_details) - ad_text = self.markets.format_ad(asset, currency, payment_details_text) - min_amount, max_amount = self.money.get_minmax(self.platform, asset, currency) - - price_formula = f"coingecko{asset.lower()}usd*usd{currency.lower()}*{settings.Agora.Margin}" - - form = { - "country_code": countrycode, - "currency": currency, - "trade_type": "ONLINE_SELL", - "asset": asset, - "price_equation": price_formula, - "track_max_amount": False, - "require_trusted_by_advertiser": False, - "online_provider": provider, - "payment_method_details": settings.Platform.PaymentMethodDetails, - "require_feedback_score": int(settings.Agora.FeedbackScore), - } - if visible is False: - form["visible"] = False - elif visible is True: - form["visible"] = False - if payment_details: - form["account_info"] = payment_details_text - form["msg"] = ad_text - form["min_amount"] = min_amount - form["max_amount"] = max_amount - - if edit: - ad = self.agora.ad(ad_id=ad_id, **form) - else: - ad = self.agora.ad_create(**form) - return ad - - def dist_countries(self, filter_asset=None): - """ - Distribute our advert into all countries and providers listed in the config. - Exits on errors. - :return: False or dict with response - :rtype: bool or dict - """ - dist_list = list(self.markets.create_distribution_list(self.platform, filter_asset)) - our_ads = self.enum_ads() - ( - supported_currencies, - account_info, - ) = self.markets.get_valid_account_details(self.platform) - # Let's get rid of the ad IDs and make it a tuple like dist_list - our_ads = [(x[0], x[2], x[3], x[4]) for x in our_ads] - for asset, countrycode, currency, provider in dist_list: - if (asset, countrycode, currency, provider) not in our_ads: - if currency in supported_currencies: - # Create the actual ad and pass in all the stuff - rtrn = self.create_ad( - asset, - countrycode, - currency, - provider, - payment_details=account_info[currency], - ) - # Bail on first error, let's not continue - if rtrn is False: - return False - yield rtrn - - def redist_countries(self): - """ - Redistribute our advert details into all our listed adverts. - This will edit all ads and update the details. Only works if we have already run dist. - This will not post any new ads. - Exits on errors. - :return: False or dict with response - :rtype: bool or dict - """ - our_ads = self.enum_ads() - ( - supported_currencies, - account_info, - ) = self.markets.get_valid_account_details(self.platform) - if not our_ads: - self.log.error("Could not get our ads.") - return False - for asset, ad_id, countrycode, currency, provider in our_ads: - if currency in supported_currencies: - rtrn = self.create_ad( - asset, - countrycode, - currency, - provider, - payment_details=account_info[currency], - edit=True, - ad_id=ad_id, - ) - # Bail on first error, let's not continue - if rtrn is False: - return False - yield (rtrn, ad_id) - - @util.handle_exceptions - def strip_duplicate_ads(self): - """ - Remove duplicate ads. - :return: list of duplicate ads - :rtype: list - """ - existing_ads = self.enum_ads() - _size = len(existing_ads) - repeated = [] - for i in range(_size): - k = i + 1 - for j in range(k, _size): - if existing_ads[i] == existing_ads[j] and existing_ads[i] not in repeated: - repeated.append(existing_ads[i]) - - actioned = [] - for ad_id, country, currency in repeated: - rtrn = self.agora.ad_delete(ad_id) - actioned.append(rtrn["success"]) - - return all(actioned) - @util.handle_exceptions def release_funds(self, contact_id): """ @@ -639,11 +38,11 @@ class Agora(util.Base): :return: response dict :rtype: dict """ - if settings.Agora.Dummy == "1": + if self.sets.Dummy == "1": self.log.error(f"Running in dummy mode, not releasing funds for {contact_id}") return - payload = {"tradeId": contact_id, "password": settings.Agora.Pass} - rtrn = self.agora._api_call( + payload = {"tradeId": contact_id, "password": self.sets.Pass} + rtrn = self.api._api_call( api_method=f"contact_release/{contact_id}", http_method="POST", query_values=payload, @@ -718,10 +117,10 @@ class Agora(util.Base): } send_cast["address"] = settings.XMR.Wallet1 - rtrn1 = self.agora.wallet_send_xmr(**send_cast) + rtrn1 = self.api.wallet_send_xmr(**send_cast) send_cast["address"] = settings.XMR.Wallet2 - rtrn2 = self.agora.wallet_send_xmr(**send_cast) + rtrn2 = self.api.wallet_send_xmr(**send_cast) self.irc.sendmsg(f"Withdrawal: {rtrn1['success']} | {rtrn2['success']}") self.ux.notify.notify_withdrawal(half_rounded) diff --git a/handler/sources/local.py b/handler/sources/local.py index e69de29..3b86112 100644 --- a/handler/sources/local.py +++ b/handler/sources/local.py @@ -0,0 +1,658 @@ +# Twisted/Klein imports +from twisted.internet.task import LoopingCall +from twisted.internet.defer import inlineCallbacks + +# Other library imports +from json import loads +from datetime import datetime +from time import sleep # TODO: async + + +# Project imports +from settings import settings +import util +from lib.agoradesk_py import AgoraDesk +from lib.localbitcoins_py import LocalBitcoins + + +class Local(util.Base): + """ + Initialise the Local API library for LBTC and Agora. + """ + + def __init__(self): + super().__init__() + if self.platform == "agora": + self.api = AgoraDesk(settings.Agora.Token) + self.sets = settings.Agora + elif self.platform == "lbtc": + self.api = LocalBitcoins(settings.LocalBitcoins.Token, settings.LocalBitcoins.Secret) + self.sets = settings.LocalBitcoins + else: + self.log.error("Platform not defined.") + + def setup_loop(self): + """ + Set up the LoopingCall to get all active trades and messages. + """ + self.log.debug("Setting up loops.") + self.lc_dash = LoopingCall(self.loop_check) + self.lc_dash.start(int(self.sets.RefreshSec)) + if settings.Agora.Cheat == "1": + self.lc_cheat = LoopingCall(self.run_cheat_in_thread) + self.lc_cheat.start(int(self.sets.CheatSec)) + self.log.debug("Finished setting up loops.") + + @inlineCallbacks + def got_dashboard(self, dash): + dash_tmp = yield self.wrap_dashboard(dash) + self.dashboard_hook(dash_tmp) + + @inlineCallbacks + def wrap_dashboard(self, dash=None): # backwards compatibility with TX + if not dash: + dash = yield self.api.dashboard() + # if dash["response"] is None: + # return False + dash_tmp = {} + if not dash: + return False + if not dash["response"]: + return False + if "data" not in dash["response"]: + # self.log.error(f"Data not in dashboard response: {dash}") + return dash_tmp + if dash["response"]["data"]["contact_count"] > 0: + for contact in dash["response"]["data"]["contact_list"]: + contact_id = contact["data"]["contact_id"] + dash_tmp[contact_id] = contact + return dash_tmp + + def loop_check(self): + """ + Calls hooks to parse dashboard info and get all contact messages. + """ + d = self.api.dashboard() + d.addCallback(self.got_dashboard) + + # Get recent messages + m = self.api.recent_messages() + m.addCallback(self.got_recent_messages) + + @inlineCallbacks + def get_dashboard_irc(self): + """ + Get dashboard helper for IRC only. + """ + dash = yield self.wrap_dashboard() + rtrn = [] + if dash is False: + return False + for contact_id, contact in dash.items(): + reference = self.tx.tx_to_ref(contact_id) + buyer = contact["data"]["buyer"]["username"] + amount = contact["data"]["amount"] + if self.platform == "agora": + asset = contact["data"]["advertisement"]["asset"] + elif self.platform == "lbtc": + asset = "BTC" + if asset == "XMR": + amount_crypto = contact["data"]["amount_xmr"] + elif asset == "BTC": + amount_crypto = contact["data"]["amount_btc"] + currency = contact["data"]["currency"] + provider = contact["data"]["advertisement"]["payment_method"] + if not contact["data"]["is_selling"]: + continue + rtrn.append( + ( + f"[#] [{reference}] ({self.platform}) <{buyer}>" + f" {amount}{currency} {provider} {amount_crypto}{asset}" + ) + ) + return rtrn + + def dashboard_hook(self, dash): + """ + Get information about our open trades. + Post new trades to IRC and cache trades for the future. + """ + current_trades = [] + if not dash: + return + if not dash.items(): + return + for contact_id, contact in dash.items(): + reference = self.tx.tx_to_ref(str(contact_id)) + if reference: + current_trades.append(reference) + buyer = contact["data"]["buyer"]["username"] + amount = contact["data"]["amount"] + if self.platform == "agora": + asset = contact["data"]["advertisement"]["asset"] + elif self.platform == "lbtc": + asset = "BTC" + provider = contact["data"]["advertisement"]["payment_method"] + if asset == "XMR": + amount_crypto = contact["data"]["amount_xmr"] + elif asset == "BTC": + amount_crypto = contact["data"]["amount_btc"] + currency = contact["data"]["currency"] + if not contact["data"]["is_selling"]: + continue + if reference not in self.last_dash: + reference = self.tx.new_trade( + self.platform, + asset, + contact_id, + buyer, + currency, + amount, + amount_crypto, + provider, + ) + if reference: + if reference not in current_trades: + current_trades.append(reference) + # Let us know there is a new trade + self.irc.sendmsg( + ( + f"[#] [{reference}] ({self.platform}) <{buyer}>" + f" {amount}{currency} {provider} {amount_crypto}{asset}" + ) + ) + # Note that we have seen this reference + self.last_dash.add(reference) + + # Purge old trades from cache + for ref in list(self.last_dash): # We're removing from the list on the fly + if ref not in current_trades: + self.last_dash.remove(ref) + if reference and reference not in current_trades: + current_trades.append(reference) + self.tx.cleanup(self.platform, current_trades) + + def got_recent_messages(self, messages, send_irc=True): + """ + Get recent messages. + """ + messages_tmp = {} + if not messages: + return False + if not messages["success"]: + return False + if "data" not in messages["response"]: + self.log.error(f"Data not in messages response: {messages['response']}") + return False + open_tx = self.tx.get_ref_map().keys() + for message in messages["response"]["data"]["message_list"]: + contact_id = message["contact_id"] + username = message["sender"]["username"] + msg = message["msg"] + if contact_id not in open_tx: + continue + reference = self.tx.tx_to_ref(contact_id) + if reference in messages_tmp: + messages_tmp[reference].append([username, msg]) + else: + messages_tmp[reference] = [[username, msg]] + + # Send new messages on IRC + if send_irc: + for user, message in messages_tmp[reference][::-1]: + if reference in self.last_messages: + if not [user, message] in self.last_messages[reference]: + self.irc.sendmsg(f"[{reference}] ({self.platform}) <{user}> {message}") + # Append sent messages to last_messages so we don't send them again + self.last_messages[reference].append([user, message]) + else: + self.last_messages[reference] = [[user, message]] + for x in messages_tmp[reference]: + self.irc.sendmsg(f"[{reference}] ({self.platform}) <{user}> {message}") + + # Purge old trades from cache + for ref in list(self.last_messages): # We're removing from the list on the fly + if ref not in messages_tmp: + del self.last_messages[ref] + + return messages_tmp + + @inlineCallbacks + def enum_ad_ids(self, page=0): + if self.platform == "lbtc" and page == 0: + page = 1 + ads = yield self.api.ads(page=page) + # ads = yield self.api._api_call(api_method="ads", query_values={"page": page}) + if ads is False: + return False + ads_total = [] + if not ads["success"]: + return False + for ad in ads["response"]["data"]["ad_list"]: + ads_total.append(ad["data"]["ad_id"]) + if "pagination" in ads["response"]: + if "next" in ads["response"]["pagination"]: + page += 1 + ads_iter = yield self.enum_ad_ids(page) + if ads_iter is None: + return False + if ads_iter is False: + return False + for ad in ads_iter: + ads_total.append(ad) + return ads_total + + @inlineCallbacks + def enum_ads(self, requested_asset=None, page=0): + if self.platform == "lbtc" and page == 0: + page = 1 + query_values = {"page": page} + if requested_asset: + query_values["asset"] = requested_asset + # ads = yield self.api._api_call(api_method="ads", query_values=query_values) + ads = yield self.api.ads(page=page) + if ads is False: + return False + ads_total = [] + if not ads["success"]: + return False + for ad in ads["response"]["data"]["ad_list"]: + if self.platform == "agora": + asset = ad["data"]["asset"] + elif self.platform == "lbtc": + asset = "BTC" + ad_id = ad["data"]["ad_id"] + country = ad["data"]["countrycode"] + currency = ad["data"]["currency"] + provider = ad["data"]["online_provider"] + ads_total.append([asset, ad_id, country, currency, provider]) + if "pagination" in ads["response"]: + if "next" in ads["response"]["pagination"]: + page += 1 + ads_iter = yield self.enum_ads(requested_asset, page) + if ads_iter is None: + return False + if ads_iter is False: + return False + for ad in ads_iter: + ads_total.append([ad[0], ad[1], ad[2], ad[3], ad[4]]) + return ads_total + + @inlineCallbacks + def enum_public_ads(self, asset, currency, providers=None, page=0): + if self.platform == "lbtc" and page == 0: + page = 1 + to_return = [] + # if asset == "XMR": + # coin = "monero" + # elif asset == "BTC": + # coin = "bitcoins" + # if not providers: + # providers = ["NATIONAL_BANK"] + # buy-monero-online, buy-bitcoin-online + # Work around Agora weirdness calling it bitcoins + # ads = yield self.api._api_call( + # api_method=f"buy-{coin}-online/{currency}", + # query_values={"page": page}, + # ) + if asset == "XMR": + ads = yield self.api.buy_monero_online(currency_code=currency, page=page) + elif asset == "BTC": + ads = yield self.api.buy_bitcoins_online(currency_code=currency, page=page) + # with open("pub.json", "a") as f: + # import json + # f.write(json.dumps([page, currency, asset, ads])+"\n") + # f.close() + if ads is None: + return False + if ads is False: + return False + if ads["response"] is None: + return False + if "data" not in ads["response"]: + return False + for ad in ads["response"]["data"]["ad_list"]: + if ad["data"]["online_provider"] not in providers: + continue + date_last_seen = ad["data"]["profile"]["last_online"] + # Check if this person was seen recently + if not util.last_online_recent(date_last_seen): + continue + ad_id = ad["data"]["ad_id"] + username = ad["data"]["profile"]["username"] + temp_price = ad["data"]["temp_price"] + provider = ad["data"]["online_provider"] + if ad["data"]["currency"] != currency: + continue + to_append = [ad_id, username, temp_price, provider, asset, currency] + if to_append not in to_return: + to_return.append(to_append) + # yield [ad_id, username, temp_price, provider, asset, currency] + if "pagination" in ads["response"]: + if "next" in ads["response"]["pagination"]: + page += 1 + ads_iter = yield self.enum_public_ads(asset, currency, providers, page) + if ads_iter is None: + return False + if ads_iter is False: + return False + for ad in ads_iter: + to_append = [ad[0], ad[1], ad[2], ad[3], ad[4], ad[5]] + if to_append not in to_return: + to_return.append(to_append) + return to_return + + def run_cheat_in_thread(self, assets=None): + """ + Update prices in another thread. + """ + if not assets: + all_assets = loads(self.sets.AssetList) + assets_not_run = set(all_assets) ^ set(self.cheat_run_on) + if not assets_not_run: + self.cheat_run_on = [] + asset = list(all_assets).pop() + self.cheat_run_on.append(asset) + else: + asset = assets_not_run.pop() + self.cheat_run_on.append(asset) + self.update_prices([asset]) + return asset + else: + # deferToThread(self.update_prices, assets) + self.update_prices(assets) + + @inlineCallbacks + def update_prices(self, assets=None): + # Get all public ads for the given assets + public_ads = yield self.get_all_public_ads(assets) + if not public_ads: + return False + + # Get the ads to update + to_update = self.markets.get_new_ad_equations(self.platform, public_ads, assets) + self.slow_ad_update(to_update) + + @inlineCallbacks + def get_all_public_ads(self, assets=None, currencies=None, providers=None): + """ + Get all public ads for our listed currencies. + :return: dict of public ads keyed by currency + :rtype: dict + """ + public_ads = {} + crypto_map = { + "XMR": "monero", + "BTC": "bitcoin", + } + + if not assets: + assets = self.markets.get_all_assets(self.platform) + # Get all currencies we have ads for, deduplicated + if not currencies: + currencies = self.markets.get_all_currencies(self.platform) + if not providers: + providers = self.markets.get_all_providers(self.platform) + sinks_currencies = self.sinks.currencies + supported_currencies = [currency for currency in currencies if currency in sinks_currencies] + currencies = supported_currencies + # We want to get the ads for each of these currencies and return the result + rates = self.money.cg.get_price(ids=["monero", "bitcoin"], vs_currencies=currencies) + for asset in assets: + for currency in currencies: + cg_asset_name = crypto_map[asset] + try: + rates[cg_asset_name][currency.lower()] + except KeyError: + # self.log.error("Error getting public ads for currency {currency}", currency=currency) + continue + ads_list = yield self.enum_public_ads(asset, currency, providers) + if not ads_list: + continue + ads = self.money.lookup_rates(self.platform, ads_list, rates=rates) + if not ads: + continue + self.write_to_es_ads("ads", ads) + if currency in public_ads: + for ad in list(ads): + if ad not in public_ads[currency]: + public_ads[currency].append(ad) + else: + public_ads[currency] = ads + + return public_ads + + def write_to_es_ads(self, msgtype, ads): + if settings.ES.Enabled == "1": + for ad in ads: + cast = { + "id": ad[0], + "username": ad[1], + "price": ad[2], + "provider": ad[3], + "asset": ad[4], + "currency": ad[5], + "margin": ad[6], + } + cast["type"] = msgtype + cast["ts"] = str(datetime.now().isoformat()) + cast["xtype"] = "platorm" + cast["market"] = self.platform + self.es.index(index=settings.ES.MetaIndex, document=cast) + + @inlineCallbacks + def slow_ad_update(self, ads): + """ + Slow ad equation update utilising exponential backoff in order to guarantee all ads are updated. + :param ads: our list of ads + """ + iterations = 0 + throttled = 0 + assets = set() + currencies = set() + while not all([x[4] for x in ads]) or iterations == 1000: + for ad_index in range(len(ads)): + ad_id, new_formula, asset, currency, actioned = ads[ad_index] + assets.add(asset) + currencies.add(currency) + if not actioned: + rtrn = yield self.api.ad_equation(ad_id, new_formula) + if rtrn["success"]: + ads[ad_index][4] = True + throttled = 0 + continue + else: + if "error_code" not in rtrn["response"]["error"]: + self.log.error(f"Error code not in return for ad {ad_id}: {rtrn['response']}") + return + if rtrn["response"]["error"]["error_code"] == 429: + throttled += 1 + sleep_time = pow(throttled, float(self.sets.SleepExponent)) + self.log.info( + f"Throttled {throttled} times while updating {ad_id}, sleeping for {sleep_time} seconds" + ) + # We're running in a thread, so this is fine + sleep(sleep_time) + self.log.error(f"Error updating ad {ad_id}: {rtrn['response']}") + continue + iterations += 1 + + @inlineCallbacks + def nuke_ads(self): + """ + Delete all of our adverts. + :return: True or False + :rtype: bool + """ + ads = yield self.enum_ad_ids() + return_ids = [] + if ads is False: + return False + for ad_id in ads: + rtrn = yield self.api.ad_delete(ad_id) + return_ids.append(rtrn["success"]) + return all(return_ids) + + @inlineCallbacks + def create_ad( + self, + asset, + countrycode, + currency, + provider, + payment_details, + visible=None, + edit=False, + ad_id=None, + ): + """ + Post an ad with the given asset in a country with a given currency. + Convert the min and max amounts from settings to the given currency with CurrencyRates. + :param asset: the crypto asset to list (XMR or BTC) + :type asset: string + :param countrycode: country code + :param currency: currency code + :param payment_details: the payment details + :type countrycode: string + :type currency: string + :type payment_details: dict + :return: data about created object or error + :rtype: dict + """ + + if payment_details: + payment_details_text = self.markets.format_payment_details(currency, payment_details) + ad_text = self.markets.format_ad(asset, currency, payment_details_text) + min_amount, max_amount = self.money.get_minmax(self.platform, asset, currency) + if self.platform == "lbtc": + bank_name = payment_details["bank"] + + if self.platform == "agora": + price_formula = f"coingecko{asset.lower()}usd*usd{currency.lower()}*{self.sets.Margin}" + elif self.platform == "lbtc": + price_formula = f"btc_in_usd*{self.sets.Margin}*USD_in_{currency}" + + form = { + "country_code": countrycode, + "currency": currency, + "trade_type": "ONLINE_SELL", + # "asset": asset, + "price_equation": price_formula, + "track_max_amount": False, + "require_trusted_by_advertiser": False, + "online_provider": provider, + "require_feedback_score": int(self.sets.FeedbackScore), + } + if self.platform == "agora": + form["asset"] = asset + form["payment_method_details"] = (settings.Platform.PaymentMethodDetails,) + form["online_provider"] = provider + elif self.platform == "lbtc": + form["online_provider"] = self.map_provider(provider, reverse=True) + + if visible is False: + form["visible"] = False + elif visible is True: + form["visible"] = False + if payment_details: + form["account_info"] = payment_details_text + form["msg"] = ad_text + form["min_amount"] = round(min_amount, 2) + form["max_amount"] = round(max_amount, 2) + if self.platform == "lbtc": + form["bank_name"] = bank_name + + if edit: + ad = self.api.ad(ad_id=ad_id, **form) + else: + ad = self.api.ad_create(**form) + return ad + + # TODO: make async + def dist_countries(self, filter_asset=None): + """ + Distribute our advert into all countries and providers listed in the config. + Exits on errors. + :return: False or dict with response + :rtype: bool or dict + """ + dist_list = list(self.markets.create_distribution_list(self.platform, filter_asset)) + our_ads = self.enum_ads() + ( + supported_currencies, + account_info, + ) = self.markets.get_valid_account_details(self.platform) + # Let's get rid of the ad IDs and make it a tuple like dist_list + our_ads = [(x[0], x[2], x[3], x[4]) for x in our_ads] + for asset, countrycode, currency, provider in dist_list: + if (asset, countrycode, currency, provider) not in our_ads: + if currency in supported_currencies: + # Create the actual ad and pass in all the stuff + rtrn = self.create_ad( + asset, + countrycode, + currency, + provider, + payment_details=account_info[currency], + ) + # Bail on first error, let's not continue + if rtrn is False: + return False + yield rtrn + + # TODO: make async + def redist_countries(self): + """ + Redistribute our advert details into all our listed adverts. + This will edit all ads and update the details. Only works if we have already run dist. + This will not post any new ads. + Exits on errors. + :return: False or dict with response + :rtype: bool or dict + """ + our_ads = self.enum_ads() + ( + supported_currencies, + account_info, + ) = self.markets.get_valid_account_details(self.platform) + if not our_ads: + self.log.error("Could not get our ads.") + return False + for asset, ad_id, countrycode, currency, provider in our_ads: + if currency in supported_currencies: + rtrn = self.create_ad( + asset, + countrycode, + currency, + provider, + payment_details=account_info[currency], + edit=True, + ad_id=ad_id, + ) + # Bail on first error, let's not continue + if rtrn is False: + return False + yield (rtrn, ad_id) + + @inlineCallbacks + def strip_duplicate_ads(self): + """ + Remove duplicate ads. + :return: list of duplicate ads + :rtype: list + """ + existing_ads = yield self.enum_ads() + _size = len(existing_ads) + repeated = [] + for i in range(_size): + k = i + 1 + for j in range(k, _size): + if existing_ads[i] == existing_ads[j] and existing_ads[i] not in repeated: + repeated.append(existing_ads[i]) + + actioned = [] + for ad_id, country, currency in repeated: + rtrn = yield self.api.ad_delete(ad_id) + actioned.append(rtrn["success"]) + + return all(actioned) diff --git a/handler/sources/localbitcoins.py b/handler/sources/localbitcoins.py index 3303060..b891f65 100644 --- a/handler/sources/localbitcoins.py +++ b/handler/sources/localbitcoins.py @@ -1,22 +1,13 @@ -# Twisted/Klein imports -from twisted.internet.task import LoopingCall -from twisted.internet.defer import inlineCallbacks - # Other library imports -from json import loads -from lib.localbitcoins_py import LocalBitcoins -from time import sleep from pyotp import TOTP -from datetime import datetime # Project imports from settings import settings import util - -# import sources.local +import sources.local -class LBTC(util.Base): +class LBTC(sources.local.Local): """ LocalBitcoins API handler. """ @@ -28,7 +19,6 @@ class LBTC(util.Base): """ self.platform = "lbtc" super().__init__() - self.lbtc = LocalBitcoins(settings.LocalBitcoins.Token, settings.LocalBitcoins.Secret) # Cache for detecting new trades self.last_dash = set() @@ -39,236 +29,6 @@ class LBTC(util.Base): # Assets that cheat has been run on self.cheat_run_on = [] - def setup_loop(self): # TODO: move to main sources - """ - Set up the LoopingCall to get all active trades and messages. - """ - self.log.debug("Setting up loops.") - self.lc_dash = LoopingCall(self.loop_check) - self.lc_dash.start(int(settings.LocalBitcoins.RefreshSec)) - if settings.LocalBitcoins.Cheat == "1": - self.lc_cheat = LoopingCall(self.run_cheat_in_thread) - self.lc_cheat.start(int(settings.LocalBitcoins.CheatSec)) - self.log.debug("Finished setting up loops.") - - @inlineCallbacks - def got_dashboard(self, dash): - dash_tmp = yield self.wrap_dashboard(dash) - self.dashboard_hook(dash_tmp) - - @inlineCallbacks - def wrap_dashboard(self, dash=None): # backwards compatibility with TX - if not dash: - dash = yield self.lbtc.dashboard() # no dashboard_seller for lbtc - # if dash["response"] is None: - # return False - dash_tmp = {} - if not dash: - return False - if not dash["response"]: - return False - if "data" not in dash["response"]: - self.log.error(f"Data not in dashboard response: {dash}") - return dash_tmp - if dash["response"]["data"]["contact_count"] > 0: - for contact in dash["response"]["data"]["contact_list"]: - contact_id = str(contact["data"]["contact_id"]) - dash_tmp[contact_id] = contact - return dash_tmp - - def loop_check(self): - """ - Calls hooks to parse dashboard info and get all contact messages. - """ - d = self.lbtc.dashboard() - d.addCallback(self.got_dashboard) - - # Get recent messages - m = self.lbtc.recent_messages() - m.addCallback(self.got_recent_messages) - - @inlineCallbacks - def get_dashboard_irc(self): - """ - Get dashboard helper for IRC only. - """ - dash = yield self.wrap_dashboard() - rtrn = [] - if dash is False: - return False - for contact_id, contact in dash.items(): - reference = self.tx.tx_to_ref(contact_id) - buyer = contact["data"]["buyer"]["username"] - amount = contact["data"]["amount"] - # asset = contact["data"]["advertisement"]["asset"] - asset = "BTC" - amount_crypto = contact["data"]["amount_btc"] - currency = contact["data"]["currency"] - provider = contact["data"]["advertisement"]["payment_method"] - if not contact["data"]["is_selling"]: - continue - rtrn.append( - ( - f"[#] [{reference}] ({self.platform}) <{buyer}>" - f" {amount}{currency} {provider} {amount_crypto}{asset}" - ) - ) - return rtrn - - def dashboard_hook(self, dash): - """ - Get information about our open trades. - Post new trades to IRC and cache trades for the future. - """ - current_trades = [] - if not dash: - return - if not dash.items(): - return - for contact_id, contact in dash.items(): - reference = self.tx.tx_to_ref(contact_id) - if reference: - current_trades.append(reference) - buyer = contact["data"]["buyer"]["username"] - amount = contact["data"]["amount"] - # asset = contact["data"]["advertisement"]["asset"] - asset = "BTC" - provider = contact["data"]["advertisement"]["payment_method"] - amount_crypto = contact["data"]["amount_btc"] - currency = contact["data"]["currency"] - if not contact["data"]["is_selling"]: - continue - if reference not in self.last_dash: - reference = self.tx.new_trade( - self.platform, - asset, - contact_id, - buyer, - currency, - amount, - amount_crypto, - provider, - ) - if reference: - if reference not in current_trades: - current_trades.append(reference) - # Let us know there is a new trade - self.irc.sendmsg( - ( - f"[#] [{reference}] ({self.platform}) <{buyer}>" - f" {amount}{currency} {provider} {amount_crypto}{asset}" - ) - ) - # Note that we have seen this reference - self.last_dash.add(reference) - # Purge old trades from cache - for ref in list(self.last_dash): # We're removing from the list on the fly - if ref not in current_trades: - self.last_dash.remove(ref) - if reference and reference not in current_trades: - current_trades.append(reference) - self.tx.cleanup(self.platform, current_trades) - - @util.handle_exceptions - def got_recent_messages(self, messages, send_irc=True): - """ - Get recent messages. - """ - messages_tmp = {} - if not messages: - return False - if not messages["success"]: - return False - if not messages["response"]: - return False - if "data" not in messages["response"]: - self.log.error(f"Data not in messages response: {messages['response']}") - return False - open_tx = self.tx.get_ref_map().keys() - for message in messages["response"]["data"]["message_list"]: - contact_id = str(message["contact_id"]) - username = message["sender"]["username"] - msg = message["msg"] - if contact_id not in open_tx: - continue - reference = self.tx.tx_to_ref(contact_id) - if reference in messages_tmp: - messages_tmp[reference].append([username, msg]) - else: - messages_tmp[reference] = [[username, msg]] - - # Send new messages on IRC - if send_irc: - for user, message in messages_tmp[reference][::-1]: - if reference in self.last_messages: - if not [user, message] in self.last_messages[reference]: - self.irc.sendmsg(f"[{reference}] ({self.platform}) <{user}> {message}") - # Append sent messages to last_messages so we don't send them again - self.last_messages[reference].append([user, message]) - else: - self.last_messages[reference] = [[user, message]] - for x in messages_tmp[reference]: - self.irc.sendmsg(f"[{reference}] ({self.platform}) <{user}> {message}") - - # Purge old trades from cache - for ref in list(self.last_messages): # We're removing from the list on the fly - if ref not in messages_tmp: - del self.last_messages[ref] - - return messages_tmp - - @inlineCallbacks - def enum_ad_ids(self, page=1): - ads = yield self.lbtc._api_call(api_method="api/ads/", query_values={"page": page}) - if ads is False: - return False - ads_total = [] - if not ads["success"]: - return False - for ad in ads["response"]["data"]["ad_list"]: - ads_total.append(ad["data"]["ad_id"]) - if "pagination" in ads["response"]: - if "next" in ads["response"]["pagination"]: - page += 1 - ads_iter = yield self.enum_ad_ids(page) - if ads_iter is None: - return False - if ads_iter is False: - return False - for ad in ads_iter: - ads_total.append(ad) - return ads_total - - @inlineCallbacks - def enum_ads(self, requested_asset=None, page=1): - query_values = {"page": page} - if requested_asset: - query_values["asset"] = requested_asset - ads = yield self.lbtc._api_call(api_method="api/ads/", query_values=query_values) - if ads is False: - return False - ads_total = [] - if not ads["success"]: - return False - for ad in ads["response"]["data"]["ad_list"]: - asset = "BTC" - ad_id = ad["data"]["ad_id"] - country = ad["data"]["countrycode"] - currency = ad["data"]["currency"] - provider = ad["data"]["online_provider"] - ads_total.append([asset, ad_id, country, currency, provider]) - if "pagination" in ads["response"]: - if "next" in ads["response"]["pagination"]: - page += 1 - ads_iter = yield self.enum_ads(requested_asset, page) - if ads_iter is None: - return False - if ads_iter is False: - return False - for ad in ads_iter: - ads_total.append([ad[0], ad[1], ad[2], ad[3], ad[4]]) - return ads_total - def map_provider(self, provider, reverse=False): provider_map = {"NATIONAL_BANK": "national-bank-transfer"} if reverse: @@ -282,363 +42,6 @@ class LBTC(util.Base): except KeyError: return False - @inlineCallbacks - def enum_public_ads(self, asset, currency, providers=None, page=1): - to_return = [] - if not providers: - providers = ["NATIONAL_BANK"] - if len(providers) == 1: - provider = providers[0] - ads = yield self.lbtc._api_call( - api_method=f"buy-bitcoins-online/{currency}/{provider}/.json", - query_values={"page": page}, - ) - else: - ads = yield self.lbtc._api_call( - api_method=f"buy-bitcoins-online/{currency}/.json", - query_values={"page": page}, - ) - # buy-monero-online, buy-bitcoin-online - # Work around weirdness calling it bitcoins - - # with open("pub.json", "a") as f: - # import json - # - # f.write(json.dumps([page, currency, asset, ads]) + "\n") - # f.close() - if ads is None: - return False - if ads is False: - return False - if ads["response"] is None: - return False - if "data" not in ads["response"]: - return False - for ad in ads["response"]["data"]["ad_list"]: - if self.map_provider(ad["data"]["online_provider"]) not in providers: - continue - date_last_seen = ad["data"]["profile"]["last_online"] - # Check if this person was seen recently - if not util.last_online_recent(date_last_seen): - continue - ad_id = str(ad["data"]["ad_id"]) - username = ad["data"]["profile"]["username"] - temp_price = ad["data"]["temp_price"] - provider = ad["data"]["online_provider"] - if ad["data"]["currency"] != currency: - continue - to_append = [ad_id, username, temp_price, provider, asset, currency] - if to_append not in to_return: - to_return.append(to_append) - # yield [ad_id, username, temp_price, provider, asset, currency] - if "pagination" in ads["response"]: - if "next" in ads["response"]["pagination"]: - page += 1 - ads_iter = yield self.enum_public_ads(asset, currency, providers, page) - if ads_iter is None: - return False - if ads_iter is False: - return False - for ad in ads_iter: - to_append = [ad[0], ad[1], ad[2], ad[3], ad[4], ad[5]] - if to_append not in to_return: - to_return.append(to_append) - return to_return - - def run_cheat_in_thread(self, assets=None): - """ - Update prices in another thread. - """ - if not assets: - all_assets = loads(settings.LocalBitcoins.AssetList) - assets_not_run = set(all_assets) ^ set(self.cheat_run_on) - if not assets_not_run: - self.cheat_run_on = [] - asset = list(all_assets).pop() - self.cheat_run_on.append(asset) - else: - asset = assets_not_run.pop() - self.cheat_run_on.append(asset) - self.update_prices([asset]) - return asset - else: - self.update_prices(assets) - - @inlineCallbacks - def update_prices(self, assets=None): - # Get all public ads for the given assets - public_ads = yield self.get_all_public_ads(assets) - if not public_ads: - return False - - # Get the ads to update - to_update = self.markets.get_new_ad_equations(self.platform, public_ads, assets) - self.slow_ad_update(to_update) - - # TODO: make generic and move to markets - @inlineCallbacks - def get_all_public_ads(self, assets=None, currencies=None, providers=None): - """ - Get all public ads for our listed currencies. - :return: dict of public ads keyed by currency - :rtype: dict - """ - public_ads = {} - crypto_map = { - "BTC": "bitcoin", - } - - if not assets: - assets = self.markets.get_all_assets(self.platform) - # Get all currencies we have ads for, deduplicated - if not currencies: - currencies = self.markets.get_all_currencies(self.platform) - if not providers: - providers = self.markets.get_all_providers(self.platform) - sinks_currencies = self.sinks.currencies - supported_currencies = [currency for currency in currencies if currency in sinks_currencies] - currencies = supported_currencies - # We want to get the ads for each of these currencies and return the result - rates = self.money.cg.get_price(ids=["bitcoin"], vs_currencies=currencies) - for asset in assets: - for currency in currencies: - cg_asset_name = crypto_map[asset] - try: - rates[cg_asset_name][currency.lower()] - except KeyError: - # self.log.error("Error getting public ads for currency {currency}", currency=currency) - continue - ads_list = yield self.enum_public_ads(asset, currency, providers) - if not ads_list: - continue - ads = self.money.lookup_rates(self.platform, ads_list, rates=rates) - if not ads: - continue - self.write_to_es_ads("ads", ads) - if currency in public_ads: - for ad in list(ads): - if ad not in public_ads[currency]: - public_ads[currency].append(ad) - else: - public_ads[currency] = ads - return public_ads - - def write_to_es_ads(self, msgtype, ads): - if settings.ES.Enabled == "1": - for ad in ads: - cast = { - "id": ad[0], - "username": ad[1], - "price": ad[2], - "provider": ad[3], - "asset": ad[4], - "currency": ad[5], - "margin": ad[6], - } - cast["type"] = msgtype - cast["ts"] = str(datetime.now().isoformat()) - cast["xtype"] = "platorm" - cast["market"] = "localbitcoins" - self.es.index(index=settings.ES.MetaIndex, document=cast) - - @inlineCallbacks - def slow_ad_update(self, ads): - """ - Slow ad equation update utilising exponential backoff in order to guarantee all ads are updated. - :param ads: our list of ads - """ - iterations = 0 - throttled = 0 - assets = set() - currencies = set() - while not all([x[4] for x in ads]) or iterations == 1000: - for ad_index in range(len(ads)): - ad_id, new_formula, asset, currency, actioned = ads[ad_index] - assets.add(asset) - currencies.add(currency) - if not actioned: - rtrn = yield self.lbtc.ad_equation(ad_id, new_formula) - if rtrn["success"]: - ads[ad_index][4] = True - throttled = 0 - continue - else: - if "error_code" not in rtrn["response"]["error"]: - self.log.error(f"Error code not in return for ad {ad_id}: {rtrn['response']}") - return - if rtrn["response"]["error"]["error_code"] == 429: - throttled += 1 - sleep_time = pow( - throttled, - float(settings.LocalBitcoins.SleepExponent), - ) - self.log.info( - f"Throttled {throttled} times while updating {ad_id}, sleeping for {sleep_time} seconds" - ) - # We're running in a thread, so this is fine - sleep(sleep_time) - self.log.error(f"Error updating ad {ad_id}: {rtrn['response']}") - continue - iterations += 1 - - @util.handle_exceptions - def nuke_ads(self): - """ - Delete all of our adverts. - :return: True or False - :rtype: bool - """ - ads = self.enum_ad_ids() - return_ids = [] - if ads is False: - return False - for ad_id in ads: - rtrn = self.lbtc.ad_delete(ad_id) - return_ids.append(rtrn["success"]) - return all(return_ids) - - @util.handle_exceptions - def create_ad( - self, - asset, - countrycode, - currency, - provider, - payment_details, - visible=None, - edit=False, - ad_id=None, - ): - """ - Post an ad with the given asset in a country with a given currency. - Convert the min and max amounts from settings to the given currency with CurrencyRates. - :param asset: the crypto asset to list (XMR or BTC) - :type asset: string - :param countrycode: country code - :param currency: currency code - :param payment_details: the payment details - :type countrycode: string - :type currency: string - :type payment_details: dict - :return: data about created object or error - :rtype: dict - """ - - if payment_details: - payment_details_text = self.markets.format_payment_details(currency, payment_details) - ad_text = self.markets.format_ad(asset, currency, payment_details_text) - min_amount, max_amount = self.money.get_minmax(self.platform, asset, currency) - bank_name = payment_details["bank"] - - price_formula = f"btc_in_usd*{settings.LocalBitcoins.Margin}*USD_in_{currency}" - form = { - "country_code": countrycode, - "currency": currency, - "trade_type": "ONLINE_SELL", - "price_equation": price_formula, - "track_max_amount": False, - "require_trusted_by_advertiser": False, - "online_provider": self.map_provider(provider, reverse=True), - "display_reference": False, - } - if visible is False: - form["visible"] = False - elif visible is True: - form["visible"] = False - if payment_details: - form["account_info"] = payment_details_text - form["msg"] = ad_text - form["min_amount"] = round(min_amount, 2) - form["max_amount"] = round(max_amount, 2) - form["bank_name"] = bank_name - - if edit: - ad = self.lbtc.ad(ad_id=ad_id, **form) - else: - ad = self.lbtc.ad_create(**form) - return ad - - def dist_countries(self, filter_asset=None): - """ - Distribute our advert into all countries and providers listed in the config. - Exits on errors. - :return: False or dict with response - :rtype: bool or dict - """ - dist_list = list(self.markets.create_distribution_list(self.platform, filter_asset)) - our_ads = self.enum_ads() - ( - supported_currencies, - account_info, - ) = self.markets.get_valid_account_details(self.platform) - # Let's get rid of the ad IDs and make it a tuple like dist_list - our_ads = [(x[0], x[2], x[3], x[4]) for x in our_ads] - for asset, countrycode, currency, provider in dist_list: - if (asset, countrycode, currency, provider) not in our_ads: - if currency in supported_currencies: - # Create the actual ad and pass in all the stuff - rtrn = self.create_ad( - asset, - countrycode, - currency, - provider, - payment_details=account_info[currency], - ) - # Bail on first error, let's not continue - if rtrn is False: - return False - yield rtrn - - def redist_countries(self): - """ - Redistribute our advert details into all our listed adverts. - This will edit all ads and update the details. Only works if we have already run dist. - This will not post any new ads. - Exits on errors. - :return: False or dict with response - :rtype: bool or dict - """ - our_ads = self.enum_ads() - ( - supported_currencies, - account_info, - ) = self.markets.get_valid_account_details(self.platform) - for asset, ad_id, countrycode, currency, provider in our_ads: - if currency in supported_currencies: - rtrn = self.create_ad( - asset, - countrycode, - currency, - provider, - payment_details=account_info[currency], - edit=True, - ad_id=ad_id, - ) - # Bail on first error, let's not continue - if rtrn is False: - return False - yield (rtrn, ad_id) - - @util.handle_exceptions - def strip_duplicate_ads(self): - """ - Remove duplicate ads. - :return: list of duplicate ads - :rtype: list - """ - existing_ads = self.enum_ads() - _size = len(existing_ads) - repeated = [] - for i in range(_size): - k = i + 1 - for j in range(k, _size): - if existing_ads[i] == existing_ads[j] and existing_ads[i] not in repeated: - repeated.append(existing_ads[i]) - - actioned = [] - for ad_id, country, currency in repeated: - return all(actioned) - @util.handle_exceptions def release_funds(self, contact_id): """ @@ -648,14 +51,14 @@ class LBTC(util.Base): :return: response dict :rtype: dict """ - if settings.LocalBitcoins.Dummy == "1": + if self.sets.Dummy == "1": self.log.error(f"Running in dummy mode, not releasing funds for {contact_id}") return payload = { "tradeId": contact_id, - "password": settings.LocalBitcoins.Pass, + "password": self.sets.Pass, } - rtrn = self.lbtc._api_call( + rtrn = self.api._api_call( api_method=f"contact_release/{contact_id}", http_method="POST", query_values=payload, @@ -730,10 +133,10 @@ class LBTC(util.Base): } send_cast["address"] = settings.XMR.Wallet1 - rtrn1 = self.lbtc.wallet_send_xmr(**send_cast) + rtrn1 = self.api.wallet_send_xmr(**send_cast) send_cast["address"] = settings.XMR.Wallet2 - rtrn2 = self.lbtc.wallet_send_xmr(**send_cast) + rtrn2 = self.api.wallet_send_xmr(**send_cast) self.irc.sendmsg(f"Withdrawal: {rtrn1['success']} | {rtrn2['success']}") self.ux.notify.notify_withdrawal(half_rounded) diff --git a/handler/tests/test_agora.py b/handler/tests/test_agora.py index aadc0eb..1596e00 100644 --- a/handler/tests/test_agora.py +++ b/handler/tests/test_agora.py @@ -110,7 +110,7 @@ class TestAgora(TestCase): def test_get_all_public_ads(self): # Override enum_public_ads - self.agora.agora._api_call = self.mock_enum_public_ads_api_call + self.agora.api._api_call = self.mock_enum_public_ads_api_call util.last_online_recent = MagicMock() util.last_online_recent.return_value = True @@ -151,7 +151,7 @@ class TestAgora(TestCase): settings.settings.Agora.ProviderList = '["REVOLUT", "NATIONAL_BANK"]' # Override enum_public_ads - self.agora.agora._api_call = self.mock_enum_public_ads_api_call + self.agora.api._api_call = self.mock_enum_public_ads_api_call util.last_online_recent = MagicMock() util.last_online_recent.return_value = True @@ -166,7 +166,7 @@ class TestAgora(TestCase): def test_enum_public_ads(self): # Override enum_public_ads - self.agora.agora._api_call = self.mock_enum_public_ads_api_call + self.agora.api._api_call = self.mock_enum_public_ads_api_call util.last_online_recent = MagicMock() util.last_online_recent.return_value = True @@ -213,7 +213,7 @@ class TestAgora(TestCase): def test_lookup_rates(self): # Override enum_public_ads - self.agora.agora._api_call = self.mock_enum_public_ads_api_call + self.agora.api._api_call = self.mock_enum_public_ads_api_call util.last_online_recent = MagicMock() util.last_online_recent.return_value = True @@ -241,7 +241,7 @@ class TestAgora(TestCase): Let's test both, and additionaly specify our own rates. """ # Override enum_public_ads - self.agora.agora._api_call = self.mock_enum_public_ads_api_call + self.agora.api._api_call = self.mock_enum_public_ads_api_call util.last_online_recent = MagicMock() util.last_online_recent.return_value = True diff --git a/handler/tests/test_lbtc.py b/handler/tests/test_lbtc.py index f775a3e..9b0bfaa 100644 --- a/handler/tests/test_lbtc.py +++ b/handler/tests/test_lbtc.py @@ -54,7 +54,7 @@ class TestLBTC(TestCase): def test_get_all_public_ads(self): # Override enum_public_ads - self.lbtc.lbtc._api_call = self.mock_enum_public_ads_api_call + self.lbtc.api._api_call = self.mock_enum_public_ads_api_call util.last_online_recent = MagicMock() util.last_online_recent.return_value = True @@ -97,7 +97,7 @@ class TestLBTC(TestCase): settings.settings.LocalBitcoins.ProviderList = '["national-bank-transfer"]' # Override enum_public_ads - self.lbtc.lbtc._api_call = self.mock_enum_public_ads_api_call + self.lbtc.api._api_call = self.mock_enum_public_ads_api_call util.last_online_recent = MagicMock() util.last_online_recent.return_value = True @@ -112,7 +112,7 @@ class TestLBTC(TestCase): def test_enum_public_ads(self): # Override enum_public_ads - self.lbtc.lbtc._api_call = self.mock_enum_public_ads_api_call + self.lbtc.api._api_call = self.mock_enum_public_ads_api_call util.last_online_recent = MagicMock() util.last_online_recent.return_value = True @@ -158,7 +158,7 @@ class TestLBTC(TestCase): def test_lookup_rates(self): # Override enum_public_ads - self.lbtc.lbtc._api_call = self.mock_enum_public_ads_api_call + self.lbtc.api._api_call = self.mock_enum_public_ads_api_call util.last_online_recent = MagicMock() util.last_online_recent.return_value = True @@ -186,7 +186,7 @@ class TestLBTC(TestCase): Let's test both, and additionaly specify our own rates. """ # Override enum_public_ads - self.lbtc.lbtc._api_call = self.mock_enum_public_ads_api_call + self.lbtc.api._api_call = self.mock_enum_public_ads_api_call util.last_online_recent = MagicMock() util.last_online_recent.return_value = True