Make cheat system async

This commit is contained in:
Mark Veidemanis 2022-05-05 09:18:35 +01:00
parent 2bf05eafa7
commit 1c84277af1
Signed by: m
GPG Key ID: 5ACFCEED46C0904F
10 changed files with 97 additions and 95 deletions

View File

@ -4,7 +4,6 @@ from twisted.internet import reactor
from klein import Klein
from twisted.internet.protocol import Factory
# Other library imports
from json import dumps
from signal import signal, SIGINT
@ -114,7 +113,7 @@ if __name__ == "__main__":
class_instance.__xmerged__()
# Set up the loops to put data in ES
init_map["tx"].setup_loops()
# init_map["tx"].setup_loops()
# Run the WebApp
init_map["webapp"].app.run(settings.App.BindHost, 8080)

View File

@ -112,7 +112,7 @@ class AgoraDesk:
response = treq.post(
api_call_url,
headers=headers,
data=json.dumps(query_values),
data=json.dumps(query_values).encode("ascii"),
)
else:
# response = httpx.post(

View File

@ -1,3 +1,6 @@
# Twisted imports
from twisted.internet.defer import inlineCallbacks
# Other library imports
from pycoingecko import CoinGeckoAPI
from forex_python.converter import CurrencyRates
@ -117,17 +120,18 @@ class Money(util.Base):
return cumul
# TODO: move to money
@inlineCallbacks
def get_profit(self, trades=False):
"""
Check how much total profit we have made.
:return: profit in USD
:rtype: float
"""
total_usd = self.tx.get_total_usd()
total_usd = yield self.tx.get_total_usd()
if not total_usd:
return False
if trades:
trades_usd = self.tx.get_open_trades_usd()
trades_usd = yield self.tx.get_open_trades_usd()
total_usd += trades_usd
profit = total_usd - float(settings.Money.BaseUSD)

View File

@ -26,6 +26,9 @@ class Sinks(util.Base):
self.startup()
self.log.debug("Finished initialising subclasses.")
def all_sinks_authenticated(self): # TODO: fix
self.tx.setup_loops()
def startup(self):
"""
We NEED the other libraries, and we initialise fast, so don't make

View File

@ -56,6 +56,7 @@ class Nordigen(util.Base):
# self.get_requisitions()
d = self.get_all_account_info()
d.addCallback(self.got_all_account_info)
self.sinks.all_sinks_authenticated()
def got_all_account_info(self, account_infos):
# Filter for added accounts since we only do that for TrueLayer

View File

@ -1,6 +1,5 @@
# Twisted/Klein imports
from twisted.internet.task import LoopingCall
from twisted.internet.threads import deferToThread
from twisted.internet.defer import inlineCallbacks
# Other library imports
@ -61,16 +60,14 @@ class Agora(util.Base):
def wrap_dashboard(self, dash=None): # backwards compatibility with TX
if not dash:
dash = yield self.agora.dashboard_seller()
if dash is None:
return False
if dash is False:
return False
# if dash["response"] is None:
# return False
dash_tmp = {}
if not dash.items():
if not dash:
return False
if "data" not in dash["response"].keys():
if not dash["response"]:
return False
if "data" not in dash["response"]:
# self.log.error(f"Data not in dashboard response: {dash}")
return dash_tmp
if dash["response"]["data"]["contact_count"] > 0:
@ -182,7 +179,7 @@ class Agora(util.Base):
Get recent messages.
"""
messages_tmp = {}
if messages is False:
if not messages:
return False
if not messages["success"]:
return False
@ -222,9 +219,9 @@ class Agora(util.Base):
return messages_tmp
@util.handle_exceptions
@inlineCallbacks
def enum_ad_ids(self, page=0):
ads = self.agora._api_call(api_method="ads", query_values={"page": page})
ads = yield self.agora._api_call(api_method="ads", query_values={"page": page})
if ads is False:
return False
ads_total = []
@ -235,7 +232,7 @@ class Agora(util.Base):
if "pagination" in ads["response"]:
if "next" in ads["response"]["pagination"]:
page += 1
ads_iter = self.enum_ad_ids(page)
ads_iter = yield self.enum_ad_ids(page)
if ads_iter is None:
return False
if ads_iter is False:
@ -244,12 +241,12 @@ class Agora(util.Base):
ads_total.append(ad)
return ads_total
@util.handle_exceptions
@inlineCallbacks
def enum_ads(self, requested_asset=None, page=0):
query_values = {"page": page}
if requested_asset:
query_values["asset"] = requested_asset
ads = self.agora._api_call(api_method="ads", query_values=query_values)
ads = yield self.agora._api_call(api_method="ads", query_values=query_values)
if ads is False:
return False
ads_total = []
@ -265,7 +262,7 @@ class Agora(util.Base):
if "pagination" in ads["response"]:
if "next" in ads["response"]["pagination"]:
page += 1
ads_iter = self.enum_ads(requested_asset, page)
ads_iter = yield self.enum_ads(requested_asset, page)
if ads_iter is None:
return False
if ads_iter is False:
@ -274,7 +271,7 @@ class Agora(util.Base):
ads_total.append([ad[0], ad[1], ad[2], ad[3], ad[4]])
return ads_total
@util.handle_exceptions
@inlineCallbacks
def enum_public_ads(self, asset, currency, providers=None, page=0):
to_return = []
if asset == "XMR":
@ -285,7 +282,7 @@ class Agora(util.Base):
providers = ["REVOLUT"]
# buy-monero-online, buy-bitcoin-online
# Work around Agora weirdness calling it bitcoins
ads = self.agora._api_call(
ads = yield self.agora._api_call(
api_method=f"buy-{coin}-online/{currency}",
query_values={"page": page},
)
@ -321,7 +318,7 @@ class Agora(util.Base):
if "pagination" in ads["response"]:
if "next" in ads["response"]["pagination"]:
page += 1
ads_iter = self.enum_public_ads(asset, currency, providers, page)
ads_iter = yield self.enum_public_ads(asset, currency, providers, page)
if ads_iter is None:
return False
if ads_iter is False:
@ -346,15 +343,16 @@ class Agora(util.Base):
else:
asset = assets_not_run.pop()
self.cheat_run_on.append(asset)
deferToThread(self.update_prices, [asset])
self.update_prices([asset])
return asset
else:
deferToThread(self.update_prices, assets)
# deferToThread(self.update_prices, assets)
self.update_prices(assets)
@util.handle_exceptions
@inlineCallbacks
def update_prices(self, assets=None):
# Get all public ads for the given assets
public_ads = self.get_all_public_ads(assets)
public_ads = yield self.get_all_public_ads(assets)
if not public_ads:
return False
@ -363,7 +361,7 @@ class Agora(util.Base):
self.slow_ad_update(to_update)
# TODO: make generic and move to markets
@util.handle_exceptions
@inlineCallbacks
def get_all_public_ads(self, assets=None, currencies=None, providers=None):
"""
Get all public ads for our listed currencies.
@ -396,7 +394,7 @@ class Agora(util.Base):
except KeyError:
# self.log.error("Error getting public ads for currency {currency}", currency=currency)
continue
ads_list = self.enum_public_ads(asset, currency, providers)
ads_list = yield self.enum_public_ads(asset, currency, providers)
if not ads_list:
continue
ads = self.money.lookup_rates(self.platform, ads_list, rates=rates)
@ -430,6 +428,7 @@ class Agora(util.Base):
cast["market"] = self.platform
self.es.index(index=settings.ES.MetaIndex, document=cast)
@inlineCallbacks
def slow_ad_update(self, ads):
"""
Slow ad equation update utilising exponential backoff in order to guarantee all ads are updated.
@ -445,7 +444,7 @@ class Agora(util.Base):
assets.add(asset)
currencies.add(currency)
if not actioned:
rtrn = self.agora.ad_equation(ad_id, new_formula)
rtrn = yield self.agora.ad_equation(ad_id, new_formula)
if rtrn["success"]:
ads[ad_index][4] = True
throttled = 0
@ -674,7 +673,6 @@ class Agora(util.Base):
if not total_usd:
return False
# total_usd += total_trades_usd
# print("total_usd after trades add", total_usd)
profit_usd = total_usd - float(settings.Money.BaseUSD)
# Get the XMR -> USD exchange rate

View File

@ -1,6 +1,5 @@
# Twisted/Klein imports
from twisted.internet.task import LoopingCall
from twisted.internet.threads import deferToThread
from twisted.internet.defer import inlineCallbacks
# Other library imports
@ -61,16 +60,14 @@ class LBTC(util.Base):
def wrap_dashboard(self, dash=None): # backwards compatibility with TX
if not dash:
dash = yield self.lbtc.dashboard() # no dashboard_seller for lbtc
if dash is None:
return False
if dash is False:
return False
# if dash["response"] is None:
# return False
dash_tmp = {}
if not dash.items():
if not dash:
return False
if "data" not in dash["response"].keys():
if not dash["response"]:
return False
if "data" not in dash["response"]:
self.log.error(f"Data not in dashboard response: {dash}")
return dash_tmp
if dash["response"]["data"]["contact_count"] > 0:
@ -178,10 +175,12 @@ class LBTC(util.Base):
Get recent messages.
"""
messages_tmp = {}
if messages is False:
if not messages:
return False
if not messages["success"]:
return False
if not messages["response"]:
return False
if "data" not in messages["response"]:
self.log.error(f"Data not in messages response: {messages['response']}")
return False
@ -218,9 +217,9 @@ class LBTC(util.Base):
return messages_tmp
@util.handle_exceptions
@inlineCallbacks
def enum_ad_ids(self, page=1):
ads = self.lbtc._api_call(api_method="api/ads/", query_values={"page": page})
ads = yield self.lbtc._api_call(api_method="api/ads/", query_values={"page": page})
if ads is False:
return False
ads_total = []
@ -231,7 +230,7 @@ class LBTC(util.Base):
if "pagination" in ads["response"]:
if "next" in ads["response"]["pagination"]:
page += 1
ads_iter = self.enum_ad_ids(page)
ads_iter = yield self.enum_ad_ids(page)
if ads_iter is None:
return False
if ads_iter is False:
@ -240,12 +239,12 @@ class LBTC(util.Base):
ads_total.append(ad)
return ads_total
@util.handle_exceptions
@inlineCallbacks
def enum_ads(self, requested_asset=None, page=1):
query_values = {"page": page}
if requested_asset:
query_values["asset"] = requested_asset
ads = self.lbtc._api_call(api_method="api/ads/", query_values=query_values)
ads = yield self.lbtc._api_call(api_method="api/ads/", query_values=query_values)
if ads is False:
return False
ads_total = []
@ -261,7 +260,7 @@ class LBTC(util.Base):
if "pagination" in ads["response"]:
if "next" in ads["response"]["pagination"]:
page += 1
ads_iter = self.enum_ads(requested_asset, page)
ads_iter = yield self.enum_ads(requested_asset, page)
if ads_iter is None:
return False
if ads_iter is False:
@ -283,19 +282,19 @@ class LBTC(util.Base):
except KeyError:
return False
@util.handle_exceptions
@inlineCallbacks
def enum_public_ads(self, asset, currency, providers=None, page=1):
to_return = []
if not providers:
providers = ["NATIONAL_BANK"]
if len(providers) == 1:
provider = providers[0]
ads = self.lbtc._api_call(
ads = yield self.lbtc._api_call(
api_method=f"buy-bitcoins-online/{currency}/{provider}/.json",
query_values={"page": page},
)
else:
ads = self.lbtc._api_call(
ads = yield self.lbtc._api_call(
api_method=f"buy-bitcoins-online/{currency}/.json",
query_values={"page": page},
)
@ -335,7 +334,7 @@ class LBTC(util.Base):
if "pagination" in ads["response"]:
if "next" in ads["response"]["pagination"]:
page += 1
ads_iter = self.enum_public_ads(asset, currency, providers, page)
ads_iter = yield self.enum_public_ads(asset, currency, providers, page)
if ads_iter is None:
return False
if ads_iter is False:
@ -360,15 +359,15 @@ class LBTC(util.Base):
else:
asset = assets_not_run.pop()
self.cheat_run_on.append(asset)
deferToThread(self.update_prices, [asset])
self.update_prices([asset])
return asset
else:
deferToThread(self.update_prices, assets)
self.update_prices(assets)
@util.handle_exceptions
@inlineCallbacks
def update_prices(self, assets=None):
# Get all public ads for the given assets
public_ads = self.get_all_public_ads(assets)
public_ads = yield self.get_all_public_ads(assets)
if not public_ads:
return False
@ -377,7 +376,7 @@ class LBTC(util.Base):
self.slow_ad_update(to_update)
# TODO: make generic and move to markets
@util.handle_exceptions
@inlineCallbacks
def get_all_public_ads(self, assets=None, currencies=None, providers=None):
"""
Get all public ads for our listed currencies.
@ -409,7 +408,7 @@ class LBTC(util.Base):
except KeyError:
# self.log.error("Error getting public ads for currency {currency}", currency=currency)
continue
ads_list = self.enum_public_ads(asset, currency, providers)
ads_list = yield self.enum_public_ads(asset, currency, providers)
if not ads_list:
continue
ads = self.money.lookup_rates(self.platform, ads_list, rates=rates)
@ -442,6 +441,7 @@ class LBTC(util.Base):
cast["market"] = "localbitcoins"
self.es.index(index=settings.ES.MetaIndex, document=cast)
@inlineCallbacks
def slow_ad_update(self, ads):
"""
Slow ad equation update utilising exponential backoff in order to guarantee all ads are updated.
@ -457,7 +457,7 @@ class LBTC(util.Base):
assets.add(asset)
currencies.add(currency)
if not actioned:
rtrn = self.lbtc.ad_equation(ad_id, new_formula)
rtrn = yield self.lbtc.ad_equation(ad_id, new_formula)
if rtrn["success"]:
ads[ad_index][4] = True
throttled = 0
@ -685,7 +685,6 @@ class LBTC(util.Base):
if not total_usd:
return False
# total_usd += total_trades_usd
# print("total_usd after trades add", total_usd)
profit_usd = total_usd - float(settings.Money.BaseUSD)
# Get the XMR -> USD exchange rate

View File

@ -66,8 +66,6 @@ class TestLBTC(TestCase):
self.lbtc.markets.get_all_providers.return_value = self.all_providers
public_ads = self.lbtc.get_all_public_ads()
print("public_ads", public_ads)
# print("fake_public_ads", fake_public_ads)
self.assertDictEqual(public_ads, fake_public_ads_lbtc)
for currency, ads in public_ads.items():

View File

@ -1,6 +1,6 @@
# Twisted/Klein imports
from twisted.internet.task import LoopingCall
from twisted.internet.threads import deferToThread
from twisted.internet.defer import inlineCallbacks
# Other library imports
from json import dumps
@ -49,13 +49,14 @@ class Transactions(util.Base):
"""
Run all the balance checks that output into ES in another thread.
"""
deferToThread(self.get_total)
deferToThread(self.get_remaining)
deferToThread(self.money.get_profit)
deferToThread(self.money.get_profit, True)
deferToThread(self.get_open_trades_usd)
deferToThread(self.get_total_remaining)
deferToThread(self.get_total_with_trades)
self.get_total()
self.get_remaining()
self.money.get_profit()
self.money.get_profit(True)
self.get_open_trades_usd()
self.get_total_remaining()
self.get_total_with_trades()
def setup_loops(self):
"""
@ -259,12 +260,9 @@ class Transactions(util.Base):
"""
senders = self.get_previous_senders(platform, platform_buyer)
if senders is None: # no senders yet, assume it's valid
print("Senders is none, assuming its a valid TX!")
return True
if platform_buyer in senders:
print("Platform buyer is in senders!")
return True
print("Platform buyer is not in senders")
self.ux.notify.notify_sender_name_mismatch(reference, platform_buyer, bank_sender)
return False
@ -295,7 +293,6 @@ class Transactions(util.Base):
Return False if the trade already has a mapped transaction.
"""
existing_tx = r.hget(f"trade.{reference}", "tx")
print("existing_tx", existing_tx)
if existing_tx is None:
return None
elif existing_tx == b"":
@ -314,7 +311,6 @@ class Transactions(util.Base):
valid = self.valid_transaction(data)
if not valid:
return False
print(f"Raw transaction data: {data}")
ts = data["timestamp"]
txid = data["transaction_id"]
amount = float(data["amount"])
@ -383,7 +379,7 @@ class Transactions(util.Base):
# Check sender - we don't do anything with this yet
sender_valid = self.check_valid_sender(reference, platform, sender, platform_buyer)
print("Sender valid for trade: ", sender_valid)
self.log.info(f"Trade {reference} buyer {platform_buyer} valid: {sender_valid}")
# trade_released = self.release_map_trade(reference, txid)
# if trade_released:
# self.ux.notify.notify_complete_trade(amount, currency)
@ -426,14 +422,11 @@ class Transactions(util.Base):
Map a trade to a transaction and release if no other TX is
mapped to the same trade.
"""
print("Release map trade called", reference, tx)
stored_trade = self.get_ref(reference)
print("Stored trade", stored_trade)
if not stored_trade:
self.log.error(f"Could not get stored trade for {reference}.")
return None
tx_obj = self.get_tx(tx)
print("tx_obj", tx_obj)
if not tx_obj:
self.log.error(f"Could not get TX for {tx}.")
return None
@ -442,13 +435,11 @@ class Transactions(util.Base):
bank_sender = tx_obj["sender"]
trade_id = stored_trade["id"]
is_updated = self.update_trade_tx(reference, tx)
print("is_updated", is_updated)
if is_updated is None:
return None
elif is_updated is True:
# We mapped the trade successfully
self.release_funds(trade_id, reference)
print("Adding mapped bank sender", platform_buyer, bank_sender)
self.add_bank_sender(platform, platform_buyer, bank_sender)
return True
elif is_updated is False:
@ -523,7 +514,6 @@ class Transactions(util.Base):
def send_verification_url(self, platform, uid, trade_id):
send_setting, post_message = self.get_send_settings(platform)
if send_setting == "1":
print("SEND SETTING IS 1", platform, uid, trade_id)
auth_url = self.ux.verify.create_applicant_and_get_link(uid)
if platform == "lbtc":
auth_url = auth_url.replace("https://", "") # hack
@ -583,7 +573,6 @@ class Transactions(util.Base):
self.irc.sendmsg(f"Generated reference for {trade_id}: {reference}")
self.ux.notify.notify_new_trade(amount, currency)
uid = self.create_uid(subclass, buyer)
print("UID of new trade", uid)
verified = self.ux.verify.get_external_user_id_status(uid)
if verified != "GREEN":
self.log.info(f"UID {uid} is not verified, sending link.")
@ -718,7 +707,6 @@ class Transactions(util.Base):
for reference in refs:
ref_data = util.convert(r.hgetall(f"trade.{reference}"))
if not ref_data:
print("NOT REF DATA")
continue
if ref_data["id"] == tx:
return reference
@ -736,6 +724,7 @@ class Transactions(util.Base):
return False
return ref_data["id"]
@inlineCallbacks
def get_total_usd(self):
"""
Get total USD in all our accounts, bank and trading.
@ -743,15 +732,21 @@ class Transactions(util.Base):
:rtype float:
"""
total_sinks_usd = self.sinks.get_total_usd()
agora_wallet_xmr = self.agora.agora.wallet_balance_xmr()
agora_wallet_xmr = yield self.agora.agora.wallet_balance_xmr()
agora_wallet_btc = yield self.agora.agora.wallet_balance()
lbtc_wallet_btc = yield self.lbtc.lbtc.wallet_balance()
if not agora_wallet_xmr["success"]:
return False
agora_wallet_btc = self.agora.agora.wallet_balance()
if not agora_wallet_btc["success"]:
return False
lbtc_wallet_btc = self.lbtc.lbtc.wallet_balance()
if not lbtc_wallet_btc["success"]:
return False
if not agora_wallet_xmr["response"]:
return False
if not agora_wallet_btc["response"]:
return False
if not lbtc_wallet_btc["response"]:
return False
total_xmr_agora = agora_wallet_xmr["response"]["data"]["total"]["balance"]
total_btc_agora = agora_wallet_btc["response"]["data"]["total"]["balance"]
total_btc_lbtc = lbtc_wallet_btc["response"]["data"]["total"]["balance"]
@ -792,6 +787,7 @@ class Transactions(util.Base):
# TODO: possibly refactor this into smaller functions which don't return as much stuff
# check if this is all really needed in the corresponding withdraw function
@inlineCallbacks
def get_total(self):
"""
Get all the values corresponding to the amount of money we hold.
@ -799,15 +795,15 @@ class Transactions(util.Base):
:rtype: tuple(tuple(float, float, float), tuple(float, float), tuple(float, float))
"""
total_sinks_usd = self.sinks.get_total_usd()
agora_wallet_xmr = self.agora.agora.wallet_balance_xmr()
agora_wallet_xmr = yield self.agora.agora.wallet_balance_xmr()
if not agora_wallet_xmr["success"]:
self.log.error("Could not get Agora XMR wallet total.")
return False
agora_wallet_btc = self.agora.agora.wallet_balance()
agora_wallet_btc = yield self.agora.agora.wallet_balance()
if not agora_wallet_btc["success"]:
self.log.error("Could not get Agora BTC wallet total.")
return False
lbtc_wallet_btc = self.lbtc.lbtc.wallet_balance()
lbtc_wallet_btc = yield self.lbtc.lbtc.wallet_balance()
if not lbtc_wallet_btc["success"]:
return False
total_xmr_agora = agora_wallet_xmr["response"]["data"]["total"]["balance"]
@ -883,13 +879,14 @@ class Transactions(util.Base):
cast["xtype"] = "tx"
self.es.index(index=settings.ES.Index, document=cast)
@inlineCallbacks
def get_remaining(self):
"""
Check how much profit we need to make in order to withdraw.
:return: profit remaining in USD
:rtype: float
"""
total_usd = self.get_total_usd()
total_usd = yield self.get_total_usd()
if not total_usd:
return False
@ -908,6 +905,7 @@ class Transactions(util.Base):
created_at = contact["data"]["created_at"]
# Reformat the date how CoinGecko likes
# 2022-05-02T11:17:14+00:00
date_parsed = datetime.strptime(created_at, "%Y-%m-%dT%H:%M:%S.%fZ")
date_formatted = date_parsed.strftime("%d-%m-%Y")
@ -934,6 +932,7 @@ class Transactions(util.Base):
cumul_usd += amount_usd
return cumul_usd
@inlineCallbacks
def get_open_trades_usd(self):
"""
Get total value of open trades in USD.
@ -941,9 +940,11 @@ class Transactions(util.Base):
:rtype: float
"""
dash_agora = self.agora.wrap_dashboard()
dash_lbtc = self.lbtc.wrap_dashboard()
dash_agora = yield dash_agora
dash_lbtc = yield dash_lbtc
if dash_agora is False:
return False
dash_lbtc = self.lbtc.wrap_dashboard()
if dash_lbtc is False:
return False
@ -958,14 +959,15 @@ class Transactions(util.Base):
self.write_to_es("get_open_trades_usd", cast_es)
return cumul_usd
@inlineCallbacks
def get_total_remaining(self):
"""
Check how much profit we need to make in order to withdraw, taking into account open trade value.
:return: profit remaining in USD
:rtype: float
"""
total_usd = self.get_total_usd()
total_trades_usd = self.get_open_trades_usd()
total_usd = yield self.get_total_usd()
total_trades_usd = yield self.get_open_trades_usd()
if not total_usd:
return False
total_usd += total_trades_usd
@ -978,11 +980,12 @@ class Transactions(util.Base):
self.write_to_es("get_total_remaining", cast_es)
return remaining
@inlineCallbacks
def get_total_with_trades(self):
total_usd = self.get_total_usd()
total_usd = yield self.get_total_usd()
if not total_usd:
return False
total_trades_usd = self.get_open_trades_usd()
total_trades_usd = yield self.get_open_trades_usd()
total_with_trades = total_usd + total_trades_usd
cast_es = {
"total_with_trades": total_with_trades,

View File

@ -25,7 +25,6 @@ class Verify(util.Base):
"""
Update the authentication status of a external user ID.
"""
print("Updating verification status", external_user_id, review_status)
if review_status == "completed" and review_answer == "GREEN":
self.verification_successful(external_user_id)
@ -38,7 +37,6 @@ class Verify(util.Base):
return signature == payload_digest
def process_callback(self, content_json):
print("CONTENT JSON", content_json)
if "externalUserId" in content_json:
external_user_id = content_json["externalUserId"]
else:
@ -85,7 +83,6 @@ class Verify(util.Base):
last_name = info["info"]["lastName"]
if first_name.startswith("MR "):
first_name = first_name[3:]
print("info", info)
return (first_name, last_name)
def create_applicant_and_get_link(self, external_user_id):