# Twisted/Klein imports from twisted.internet.task import LoopingCall from twisted.internet.defer import inlineCallbacks # Other library imports from json import loads from datetime import datetime from time import sleep # TODO: async # Project imports from settings import settings import util from lib.agoradesk_py import AgoraDesk from lib.localbitcoins_py import LocalBitcoins import db from lib.logstash import send_logstash class Local(util.Base): """ Initialise the Local API library for LBTC and Agora. """ def __init__(self): super().__init__() if self.platform == "agora": self.api = AgoraDesk(settings.Agora.Token) self.sets = settings.Agora elif self.platform == "lbtc": self.api = LocalBitcoins(settings.LocalBitcoins.Token, settings.LocalBitcoins.Secret) self.sets = settings.LocalBitcoins else: self.log.error("Platform not defined.") def setup_loop(self): """ Set up the LoopingCall to get all active trades and messages. """ self.log.debug("Setting up loops.") self.lc_dash = LoopingCall(self.loop_check) self.lc_dash.start(int(self.sets.RefreshSec)) if settings.Agora.Cheat == "1": self.lc_cheat = LoopingCall(self.run_cheat_in_thread) self.lc_cheat.start(int(self.sets.CheatSec)) self.log.debug("Finished setting up loops.") def map_provider(self, provider, reverse=False): provider_map = {"NATIONAL_BANK": "national-bank-transfer"} if reverse: try: return next(key for key, value in provider_map.items() if value == provider) except StopIteration: return False else: try: return provider_map[provider] except KeyError: return False @inlineCallbacks def got_dashboard(self, dash): dash_tmp = yield self.wrap_dashboard(dash) self.dashboard_hook(dash_tmp) @inlineCallbacks def wrap_dashboard(self, dash=None): # backwards compatibility with TX if not dash: dash = yield self.api.dashboard() # if dash["response"] is None: # return False dash_tmp = {} if not dash: return False if not dash["response"]: return False if "data" not in dash["response"]: # self.log.error(f"Data not in dashboard response: {dash}") return dash_tmp if dash["response"]["data"]["contact_count"] > 0: for contact in dash["response"]["data"]["contact_list"]: contact_id = contact["data"]["contact_id"] dash_tmp[contact_id] = contact return dash_tmp def loop_check(self): """ Calls hooks to parse dashboard info and get all contact messages. """ d = self.api.dashboard() d.addCallback(self.got_dashboard) # Get recent messages m = self.api.recent_messages() m.addCallback(self.got_recent_messages) @inlineCallbacks def get_dashboard_irc(self): """ Get dashboard helper for IRC only. """ dash = yield self.wrap_dashboard() rtrn = [] if dash is False: return False for contact_id, contact in dash.items(): reference = db.tx_to_ref(contact_id) buyer = contact["data"]["buyer"]["username"] amount = contact["data"]["amount"] if self.platform == "agora": asset = contact["data"]["advertisement"]["asset"] elif self.platform == "lbtc": asset = "BTC" if asset == "XMR": amount_crypto = contact["data"]["amount_xmr"] elif asset == "BTC": amount_crypto = contact["data"]["amount_btc"] currency = contact["data"]["currency"] provider = contact["data"]["advertisement"]["payment_method"] if not contact["data"]["is_selling"]: continue rtrn.append( ( f"[#] [{reference}] ({self.platform}) <{buyer}>" f" {amount}{currency} {provider} {amount_crypto}{asset}" ) ) return rtrn def dashboard_hook(self, dash): """ Get information about our open trades. Post new trades to IRC and cache trades for the future. """ current_trades = [] if not dash: return if not dash.items(): return for contact_id, contact in dash.items(): reference = db.tx_to_ref(str(contact_id)) if reference: current_trades.append(reference) buyer = contact["data"]["buyer"]["username"] amount = contact["data"]["amount"] if self.platform == "agora": asset = contact["data"]["advertisement"]["asset"] elif self.platform == "lbtc": asset = "BTC" provider = contact["data"]["advertisement"]["payment_method"] if asset == "XMR": amount_crypto = contact["data"]["amount_xmr"] elif asset == "BTC": amount_crypto = contact["data"]["amount_btc"] currency = contact["data"]["currency"] if not contact["data"]["is_selling"]: continue if reference not in self.last_dash: reference = self.tx.new_trade( self.platform, asset, contact_id, buyer, currency, amount, amount_crypto, provider, ) if reference: if reference not in current_trades: current_trades.append(reference) # Let us know there is a new trade self.irc.sendmsg( ( f"[#] [{reference}] ({self.platform}) <{buyer}>" f" {amount}{currency} {provider} {amount_crypto}{asset}" ) ) # Note that we have seen this reference self.last_dash.add(reference) # Purge old trades from cache for ref in list(self.last_dash): # We're removing from the list on the fly if ref not in current_trades: self.last_dash.remove(ref) if reference and reference not in current_trades: current_trades.append(reference) messages = db.cleanup(self.platform, current_trades) for message in messages: self.ux.irc.sendmsg(message) def got_recent_messages(self, messages, send_irc=True): """ Get recent messages. """ messages_tmp = {} if not messages: return False if not messages["success"]: return False if not messages["response"]: return False if "data" not in messages["response"]: self.log.error(f"Data not in messages response: {messages['response']}") return False open_tx = db.get_ref_map().keys() for message in messages["response"]["data"]["message_list"]: contact_id = str(message["contact_id"]) username = message["sender"]["username"] msg = message["msg"] if contact_id not in open_tx: continue reference = db.tx_to_ref(contact_id) if reference in messages_tmp: messages_tmp[reference].append([username, msg]) else: messages_tmp[reference] = [[username, msg]] # Send new messages on IRC if send_irc: for user, message in messages_tmp[reference][::-1]: if reference in self.last_messages: if not [user, message] in self.last_messages[reference]: self.irc.sendmsg(f"[{reference}] ({self.platform}) <{user}> {message}") # Append sent messages to last_messages so we don't send them again self.last_messages[reference].append([user, message]) else: self.last_messages[reference] = [[user, message]] for x in messages_tmp[reference]: self.irc.sendmsg(f"[{reference}] ({self.platform}) <{user}> {message}") # Purge old trades from cache for ref in list(self.last_messages): # We're removing from the list on the fly if ref not in messages_tmp: del self.last_messages[ref] return messages_tmp @inlineCallbacks def enum_ad_ids(self, page=0): if self.platform == "lbtc" and page == 0: page = 1 ads = yield self.api.ads(page=page) # ads = yield self.api._api_call(api_method="ads", query_values={"page": page}) if ads is False: return False ads_total = [] if not ads["success"]: return False for ad in ads["response"]["data"]["ad_list"]: ads_total.append(ad["data"]["ad_id"]) if "pagination" in ads["response"]: if "next" in ads["response"]["pagination"]: page += 1 ads_iter = yield self.enum_ad_ids(page) if ads_iter is None: return False if ads_iter is False: return False for ad in ads_iter: ads_total.append(ad) return ads_total @inlineCallbacks def enum_ads(self, requested_asset=None, page=0): if self.platform == "lbtc" and page == 0: page = 1 query_values = {"page": page} if requested_asset: query_values["asset"] = requested_asset # ads = yield self.api._api_call(api_method="ads", query_values=query_values) ads = yield self.api.ads(page=page) if ads is False: return False ads_total = [] if not ads["success"]: return False if not ads["response"]: return False for ad in ads["response"]["data"]["ad_list"]: if self.platform == "agora": asset = ad["data"]["asset"] elif self.platform == "lbtc": asset = "BTC" ad_id = ad["data"]["ad_id"] country = ad["data"]["countrycode"] currency = ad["data"]["currency"] provider = ad["data"]["online_provider"] ads_total.append([asset, ad_id, country, currency, provider]) if "pagination" in ads["response"]: if "next" in ads["response"]["pagination"]: page += 1 ads_iter = yield self.enum_ads(requested_asset, page) if ads_iter is None: return False if ads_iter is False: return False for ad in ads_iter: ads_total.append([ad[0], ad[1], ad[2], ad[3], ad[4]]) return ads_total @inlineCallbacks def enum_public_ads(self, asset, currency, providers=None, page=0): if self.platform == "lbtc" and page == 0: page = 1 to_return = [] # if asset == "XMR": # coin = "monero" # elif asset == "BTC": # coin = "bitcoins" if not providers: providers = ["NATIONAL_BANK"] # buy-monero-online, buy-bitcoin-online # Work around Agora weirdness calling it bitcoins # ads = yield self.api._api_call( # api_method=f"buy-{coin}-online/{currency}", # query_values={"page": page}, # ) if asset == "XMR": ads = yield self.api.buy_monero_online(currency_code=currency, page=page) elif asset == "BTC": ads = yield self.api.buy_bitcoins_online(currency_code=currency, page=page) # with open("pub.json", "a") as f: # import json # f.write(json.dumps([page, currency, asset, ads])+"\n") # f.close() if ads is None: return False if ads is False: return False if ads["response"] is None: return False if "data" not in ads["response"]: return False for ad in ads["response"]["data"]["ad_list"]: provider = ad["data"]["online_provider"] if self.platform == "lbtc": provider_test = self.map_provider(provider) else: provider_test = provider if provider_test not in providers: continue date_last_seen = ad["data"]["profile"]["last_online"] # Check if this person was seen recently if not util.last_online_recent(date_last_seen): continue ad_id = str(ad["data"]["ad_id"]) username = ad["data"]["profile"]["username"] temp_price = ad["data"]["temp_price"] if ad["data"]["currency"] != currency: continue to_append = [ad_id, username, temp_price, provider, asset, currency] if to_append not in to_return: to_return.append(to_append) # yield [ad_id, username, temp_price, provider, asset, currency] if "pagination" in ads["response"]: if "next" in ads["response"]["pagination"]: page += 1 ads_iter = yield self.enum_public_ads(asset, currency, providers, page) if ads_iter is None: return False if ads_iter is False: return False for ad in ads_iter: to_append = [ad[0], ad[1], ad[2], ad[3], ad[4], ad[5]] if to_append not in to_return: to_return.append(to_append) return to_return def run_cheat_in_thread(self, assets=None): """ Update prices in another thread. """ if not assets: all_assets = loads(self.sets.AssetList) assets_not_run = set(all_assets) ^ set(self.cheat_run_on) if not assets_not_run: self.cheat_run_on = [] asset = list(all_assets).pop() self.cheat_run_on.append(asset) else: asset = assets_not_run.pop() self.cheat_run_on.append(asset) self.update_prices([asset]) return asset else: # deferToThread(self.update_prices, assets) self.update_prices(assets) @inlineCallbacks def update_prices(self, assets=None): # Get all public ads for the given assets public_ads = yield self.get_all_public_ads(assets) if not public_ads: return False # Get the ads to update to_update = self.markets.get_new_ad_equations(self.platform, public_ads, assets) self.slow_ad_update(to_update) @inlineCallbacks def get_all_public_ads(self, assets=None, currencies=None, providers=None): """ Get all public ads for our listed currencies. :return: dict of public ads keyed by currency :rtype: dict """ public_ads = {} crypto_map = { "XMR": "monero", "BTC": "bitcoin", } if not assets: assets = self.markets.get_all_assets(self.platform) # Get all currencies we have ads for, deduplicated if not currencies: currencies = self.markets.get_all_currencies(self.platform) if not providers: providers = self.markets.get_all_providers(self.platform) sinks_currencies = self.sinks.currencies supported_currencies = [currency for currency in currencies if currency in sinks_currencies] currencies = supported_currencies # We want to get the ads for each of these currencies and return the result rates = self.money.cg.get_price(ids=["monero", "bitcoin"], vs_currencies=currencies) for asset in assets: for currency in currencies: cg_asset_name = crypto_map[asset] try: rates[cg_asset_name][currency.lower()] except KeyError: self.log.debug(f"Error getting public ads for currency: {currency}") continue ads_list = yield self.enum_public_ads(asset, currency, providers) if not ads_list: self.log.debug("Error getting ads list.") continue ads = self.money.lookup_rates(self.platform, ads_list, rates=rates) if not ads: self.log.debug("Error lookup up rates.") continue self.log.debug("Writing to ES.") self.write_to_es_ads("ads", ads) if currency in public_ads: for ad in list(ads): if ad not in public_ads[currency]: public_ads[currency].append(ad) else: public_ads[currency] = ads return public_ads def write_to_es_ads(self, msgtype, ads): for ad in ads: cast = { "id": ad[0], "username": ad[1], "price": ad[2], "provider": ad[3], "asset": ad[4], "currency": ad[5], "margin": ad[6], "ts": str(datetime.now().isoformat()), "xtype": msgtype, "market": self.platform, "type": "platform", } if settings.ES.Enabled == "1": self.es.index(index=settings.ES.MetaIndex, body=cast) elif settings.Logstash.Enabled == "1": send_logstash(cast) @inlineCallbacks def slow_ad_update(self, ads): """ Slow ad equation update utilising exponential backoff in order to guarantee all ads are updated. :param ads: our list of ads """ iterations = 0 throttled = 0 assets = set() currencies = set() while not all([x[4] for x in ads]) or iterations == 1000: for ad_index in range(len(ads)): ad_id, new_formula, asset, currency, actioned = ads[ad_index] assets.add(asset) currencies.add(currency) if not actioned: rtrn = yield self.api.ad_equation(ad_id, new_formula) if rtrn["success"]: ads[ad_index][4] = True throttled = 0 continue else: if "error_code" not in rtrn["response"]["error"]: self.log.error(f"Error code not in return for ad {ad_id}: {rtrn['response']}") return if rtrn["response"]["error"]["error_code"] == 429: throttled += 1 sleep_time = pow(throttled, float(self.sets.SleepExponent)) self.log.info( f"Throttled {throttled} times while updating {ad_id}, sleeping for {sleep_time} seconds" ) # We're running in a thread, so this is fine sleep(sleep_time) self.log.error(f"Error updating ad {ad_id}: {rtrn['response']}") continue iterations += 1 @inlineCallbacks def nuke_ads(self): """ Delete all of our adverts. :return: True or False :rtype: bool """ ads = yield self.enum_ad_ids() return_ids = [] if ads is False: return False for ad_id in ads: rtrn = yield self.api.ad_delete(ad_id) return_ids.append(rtrn["success"]) return all(return_ids) @inlineCallbacks def create_ad( self, asset, countrycode, currency, provider, payment_details, visible=None, edit=False, ad_id=None, ): """ Post an ad with the given asset in a country with a given currency. Convert the min and max amounts from settings to the given currency with CurrencyRates. :param asset: the crypto asset to list (XMR or BTC) :type asset: string :param countrycode: country code :param currency: currency code :param payment_details: the payment details :type countrycode: string :type currency: string :type payment_details: dict :return: data about created object or error :rtype: dict """ if payment_details: payment_details_text = self.markets.format_payment_details(currency, payment_details) ad_text = self.markets.format_ad(asset, currency, payment_details_text) min_amount, max_amount = self.money.get_minmax(self.platform, asset, currency) if self.platform == "lbtc": bank_name = payment_details["bank"] if self.platform == "agora": price_formula = f"coingecko{asset.lower()}usd*usd{currency.lower()}*{self.sets.Margin}" elif self.platform == "lbtc": price_formula = f"btc_in_usd*{self.sets.Margin}*USD_in_{currency}" form = { "country_code": countrycode, "currency": currency, "trade_type": "ONLINE_SELL", # "asset": asset, "price_equation": price_formula, "track_max_amount": False, "require_trusted_by_advertiser": False, "online_provider": provider, "require_feedback_score": int(self.sets.FeedbackScore), } if self.platform == "agora": form["asset"] = asset form["payment_method_details"] = settings.Platform.PaymentMethodDetails form["online_provider"] = provider elif self.platform == "lbtc": form["online_provider"] = self.map_provider(provider, reverse=True) if visible is False: form["visible"] = False elif visible is True: form["visible"] = False if payment_details: form["account_info"] = payment_details_text form["msg"] = ad_text form["min_amount"] = round(min_amount, 2) form["max_amount"] = round(max_amount, 2) if self.platform == "lbtc": form["bank_name"] = bank_name if edit: ad = yield self.api.ad(ad_id=ad_id, **form) else: ad = yield self.api.ad_create(**form) return ad @inlineCallbacks def dist_countries(self, filter_asset=None): """ Distribute our advert into all countries and providers listed in the config. Exits on errors. :return: False or dict with response :rtype: bool or dict """ dist_list = list(self.markets.create_distribution_list(self.platform, filter_asset)) our_ads = yield self.enum_ads() ( supported_currencies, account_info, ) = self.markets.get_valid_account_details(self.platform) # Let's get rid of the ad IDs and make it a tuple like dist_list our_ads = [(x[0], x[2], x[3], x[4]) for x in our_ads] if not our_ads: self.log.error("Could not get our ads.") return False to_return = [] for asset, countrycode, currency, provider in dist_list: if (asset, countrycode, currency, provider) not in our_ads: if currency in supported_currencies: # Create the actual ad and pass in all the stuff rtrn = yield self.create_ad( asset, countrycode, currency, provider, payment_details=account_info[currency], ) # Bail on first error, let's not continue if rtrn is False: return False to_return.append(rtrn) return to_return @inlineCallbacks def redist_countries(self): """ Redistribute our advert details into all our listed adverts. This will edit all ads and update the details. Only works if we have already run dist. This will not post any new ads. Exits on errors. :return: False or dict with response :rtype: bool or dict """ our_ads = yield self.enum_ads() ( supported_currencies, account_info, ) = self.markets.get_valid_account_details(self.platform) if not our_ads: self.log.error("Could not get our ads.") return False to_return = [] for asset, ad_id, countrycode, currency, provider in our_ads: if currency in supported_currencies: rtrn = yield self.create_ad( asset, countrycode, currency, provider, payment_details=account_info[currency], edit=True, ad_id=ad_id, ) # Bail on first error, let's not continue if rtrn is False: return False to_return.append((rtrn, ad_id)) return to_return @inlineCallbacks def strip_duplicate_ads(self): """ Remove duplicate ads. :return: list of duplicate ads :rtype: list """ existing_ads = yield self.enum_ads() _size = len(existing_ads) repeated = [] for i in range(_size): k = i + 1 for j in range(k, _size): if existing_ads[i] == existing_ads[j] and existing_ads[i] not in repeated: repeated.append(existing_ads[i]) actioned = [] for ad_id, country, currency in repeated: rtrn = yield self.api.ad_delete(ad_id) actioned.append(rtrn["success"]) return all(actioned)