diff --git a/examples/create_cancel_order.py b/examples/create_cancel_order.py index a9e8ce5..4264e3c 100644 --- a/examples/create_cancel_order.py +++ b/examples/create_cancel_order.py @@ -6,6 +6,7 @@ # The API_KEY_PRIVATE_KEY provided belongs to a dummy account registered on Testnet. # It was generated using the setup_system.py script, and servers as an example. +# Alternatively, you can go to https://app.lighter.xyz/apikeys for mainnet api keys BASE_URL = "https://testnet.zklighter.elliot.ai" API_KEY_PRIVATE_KEY = "0xed636277f3753b6c0275f7a28c2678a7f3a95655e09deaebec15179b50c5da7f903152e50f594f7b" ACCOUNT_INDEX = 65 @@ -36,7 +37,7 @@ async def main(): market_index=0, client_order_index=123, base_amount=100000, - price=270000, + price=405000, is_ask=True, order_type=lighter.SignerClient.ORDER_TYPE_LIMIT, time_in_force=lighter.SignerClient.ORDER_TIME_IN_FORCE_GOOD_TILL_TIME, diff --git a/examples/create_with_multiple_keys.py b/examples/create_with_multiple_keys.py new file mode 100644 index 0000000..45c874c --- /dev/null +++ b/examples/create_with_multiple_keys.py @@ -0,0 +1,48 @@ +import asyncio +import lighter + + +BASE_URL = "https://testnet.zklighter.elliot.ai" +# use examples/system_setup.py or the apikeys page (for mainnet) to generate new api keys +KEYS = { + 5: "API_PRIVATE_KEY_5", + 6: "API_PRIVATE_KEY_6", + 7: "API_PRIVATE_KEY_7", +} +ACCOUNT_INDEX = 100 # replace with your account_index + + +async def main(): + client = lighter.SignerClient( + url=BASE_URL, + private_key=KEYS[5], + account_index=ACCOUNT_INDEX, + api_key_index=5, + max_api_key_index=7, + private_keys=KEYS, + ) + + err = client.check_client() + if err is not None: + print(f"CheckClient error: {err}") + return + + for i in range(20): + res_tuple = await client.create_order( + market_index=0, + client_order_index=123 + i, + base_amount=100000 + i, + price=385000 + i, + is_ask=True, + order_type=lighter.SignerClient.ORDER_TYPE_LIMIT, + time_in_force=lighter.SignerClient.ORDER_TIME_IN_FORCE_GOOD_TILL_TIME, + reduce_only=0, + trigger_price=0, + ) + print(res_tuple) + + await client.cancel_all_orders(time_in_force=client.CANCEL_ALL_TIF_IMMEDIATE, time=0) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/lighter/errors.py b/lighter/errors.py new file mode 100644 index 0000000..2851fa7 --- /dev/null +++ b/lighter/errors.py @@ -0,0 +1,2 @@ +class ValidationError(ValueError): + pass diff --git a/lighter/nonce_manager.py b/lighter/nonce_manager.py new file mode 100644 index 0000000..cf4c730 --- /dev/null +++ b/lighter/nonce_manager.py @@ -0,0 +1,132 @@ +import abc +import enum +from typing import Optional, Tuple + +import requests + +from lighter import api_client +from lighter.api import transaction_api +from lighter.errors import ValidationError + + +def get_nonce_from_api(client: api_client.ApiClient, account_index: int, api_key_index: int) -> int: + # uses request to avoid async initialization + req = requests.get( + client.configuration.host + "/api/v1/nextNonce", + params={"account_index": account_index, "api_key_index": api_key_index}, + ) + if req.status_code != 200: + raise Exception(f"couldn't get nonce {req.content}") + return req.json()["nonce"] + + +class NonceManager(abc.ABC): + def __init__( + self, + account_index: int, + api_client: api_client.ApiClient, + start_api_key: int, + end_api_key: Optional[int] = None, + ): + if end_api_key is None: + end_api_key = start_api_key + if start_api_key > end_api_key or start_api_key >= 255 or end_api_key >= 255: + raise ValidationError(f"invalid range {start_api_key=} {end_api_key=}") + self.start_api_key = start_api_key + self.end_api_key = end_api_key + self.current_api_key = end_api_key # start will be used for the first tx + self.account_index = account_index + self.api_client = api_client + self.nonce = { + api_key_index: get_nonce_from_api(api_client, account_index, api_key_index) - 1 + for api_key_index in range(start_api_key, end_api_key + 1) + } + + def hard_refresh_nonce(self, api_key: int): + self.nonce[api_key] = get_nonce_from_api(self.api_client, self.account_index, api_key) - 1 + + @abc.abstractmethod + def next_nonce(self) -> Tuple[int, int]: + pass + + def acknowledge_failure(self, api_key_index: int) -> None: + pass + + +def increment_circular(idx: int, start_idx: int, end_idx: int) -> int: + idx += 1 + if idx > end_idx: + return start_idx + return idx + + +class OptimisticNonceManager(NonceManager): + def __init__( + self, + account_index: int, + api_client: api_client.ApiClient, + start_api_key: int, + end_api_key: Optional[int] = None, + ) -> None: + super().__init__(account_index, api_client, start_api_key, end_api_key) + + def next_nonce(self) -> Tuple[int, int]: + self.current_api_key = increment_circular(self.current_api_key, self.start_api_key, self.end_api_key) + self.nonce[self.current_api_key] += 1 + return (self.current_api_key, self.nonce[self.current_api_key]) + + def acknowledge_failure(self, api_key_index: int) -> None: + self.nonce[api_key_index] -= 1 + + +class ApiNonceManager(NonceManager): + def __init__( + self, + account_index: int, + api_client: api_client.ApiClient, + start_api_key: int, + end_api_key: Optional[int] = None, + ) -> None: + super().__init__(account_index, api_client, start_api_key, end_api_key) + + def next_nonce(self) -> Tuple[int, int]: + """ + It is recommended to wait at least 350ms before using the same api key. + Please be mindful of your transaction frequency when using this nonce manager. + predicted_execution_time_ms from the response could give you a tighter bound. + """ + self.current_api_key = increment_circular(self.current_api_key, self.start_api_key, self.end_api_key) + self.nonce[self.current_api_key] = get_nonce_from_api(self.api_client, self.account_index, self.current_api_key) + return (self.current_api_key, self.nonce[self.current_api_key]) + + def refresh_nonce(self, api_key_index: int) -> int: + self.nonce[api_key_index] = get_nonce_from_api(self.api_client, self.start_api_key, self.end_api_key) + + +class NonceManagerType(enum.Enum): + OPTIMISTIC = 1 + API = 2 + + +def nonce_manager_factory( + nonce_manager_type: NonceManagerType, + account_index: int, + api_client: api_client.ApiClient, + start_api_key: int, + end_api_key: Optional[int] = None, +) -> NonceManager: + if nonce_manager_type == NonceManagerType.OPTIMISTIC: + return OptimisticNonceManager( + account_index=account_index, + api_client=api_client, + start_api_key=start_api_key, + end_api_key=end_api_key, + ) + elif nonce_manager_type == NonceManagerType.API: + return ApiNonceManager( + account_index=account_index, + api_client=api_client, + start_api_key=start_api_key, + end_api_key=end_api_key, + ) + raise ValidationError("invalid nonce manager type") diff --git a/lighter/signer_client.py b/lighter/signer_client.py index 6ebdc6b..15bf092 100644 --- a/lighter/signer_client.py +++ b/lighter/signer_client.py @@ -1,20 +1,27 @@ import ctypes +from functools import wraps +import inspect import json import platform import logging import os import time +from typing import Dict, Optional, Tuple from eth_account import Account from eth_account.messages import encode_defunct from pydantic import StrictInt import lighter from lighter.configuration import Configuration +from lighter.errors import ValidationError from lighter.models import TxHash +from lighter import nonce_manager from lighter.transactions import CreateOrder, CancelOrder, Withdraw logging.basicConfig(level=logging.DEBUG) +CODE_OK = 200 + class ApiKeyResponse(ctypes.Structure): _fields_ = [("privateKey", ctypes.c_char_p), ("publicKey", ctypes.c_char_p), ("err", ctypes.c_char_p)] @@ -60,6 +67,47 @@ def create_api_key(seed=""): return private_key_str, public_key_str, error +def trim_exc(exception_body: str): + return exception_body.strip().split("\n")[-1] + + +def process_api_key_and_nonce(func): + @wraps(func) + async def wrapper(self, *args, **kwargs): + # Get the signature + sig = inspect.signature(func) + + # Bind args and kwargs to the function's signature + bound_args = sig.bind(self, *args, **kwargs) + bound_args.apply_defaults() + # Extract api_key_index and nonce from kwargs or use defaults + api_key_index = bound_args.arguments.get("api_key_index", -1) + nonce = bound_args.arguments.get("nonce", -1) + if api_key_index == -1 and nonce == -1: + api_key_index, nonce = self.nonce_manager.next_nonce() + err = self.switch_api_key(api_key_index) + if err != None: + raise Exception(f"error switching api key: {err}") + + # Call the original function with modified kwargs + ret: TxHash + try: + created_tx, ret, err = await func(self, *args, **kwargs, nonce=nonce, api_key_index=api_key_index) + if ret.code != CODE_OK: + self.nonce_manager.acknowledge_failure(api_key_index) + except lighter.exceptions.BadRequestException as e: + if "invalid nonce" in str(e): + self.nonce_manager.hard_refresh_nonce(api_key_index) + return None, None, trim_exc(str(e)) + else: + self.nonce_manager.acknowledge_failure(api_key_index) + return None, None, trim_exc(str(e)) + + return created_tx, ret, err + + return wrapper + + class SignerClient: USDC_TICKER_SCALE = 1e6 @@ -88,13 +136,30 @@ class SignerClient: ORDER_TIME_IN_FORCE_GOOD_TILL_TIME = 1 ORDER_TIME_IN_FORCE_POST_ONLY = 2 + CANCEL_ALL_TIF_IMMEDIATE = 0 + CANCEL_ALL_TIF_SCHEDULED = 1 + CANCEL_ALL_TIF_ABORT = 2 + NIL_TRIGGER_PRICE = 0 DEFAULT_28_DAY_ORDER_EXPIRY = -1 DEFAULT_IOC_EXPIRY = 0 DEFAULT_10_MIN_AUTH_EXPIRY = -1 MINUTE = 60 - def __init__(self, url, private_key, api_key_index, account_index): + def __init__( + self, + url, + private_key, + api_key_index, + account_index, + max_api_key_index=-1, + private_keys: Optional[Dict[int, str]] = None, + nonce_management_type=nonce_manager.NonceManagerType.OPTIMISTIC, + ): + """ + First private key needs to be passed separately for backwards compatibility. + This may get deprecated in a future version. + """ chain_id = 304 if "mainnet" in url else 300 # api_key_index=0 is generally used by frontend @@ -104,13 +169,45 @@ def __init__(self, url, private_key, api_key_index, account_index): self.private_key = private_key self.chain_id = chain_id self.api_key_index = api_key_index + if max_api_key_index == -1: + self.end_api_key_index = api_key_index + else: + self.end_api_key_index = max_api_key_index + + private_keys = private_keys or {} + self.validate_api_private_keys(private_key, private_keys) + self.api_key_dict = self.build_api_key_dict(private_key, private_keys) self.account_index = account_index self.signer = _initialize_signer() self.api_client = lighter.ApiClient(configuration=Configuration(host=url)) self.tx_api = lighter.TransactionApi(self.api_client) - self.create_client() - - def create_client(self): + self.nonce_manager = nonce_manager.nonce_manager_factory( + nonce_manager_type=nonce_management_type, + account_index=account_index, + api_client=self.api_client, + start_api_key=self.api_key_index, + end_api_key=self.end_api_key_index, + ) + for api_key in range(self.api_key_index, self.end_api_key_index + 1): + self.create_client(api_key) + + def validate_api_private_keys(self, initial_private_key: str, private_keys: Dict[int, str]): + if len(private_keys) == self.end_api_key_index - self.api_key_index + 1: + if private_keys[self.api_key_index] != initial_private_key: + raise ValidationError("inconsistent private keys") + return # this is all we need to check in this case + if len(private_keys) != self.end_api_key_index - self.api_key_index: + raise ValidationError("unexpected number of private keys") + for api_key in range(self.api_key_index + 1, self.end_api_key_index): + if api_key not in private_keys: + raise Exception(f"missing {api_key=} private key!") + + def build_api_key_dict(self, private_key, private_keys): + if len(private_keys) == self.end_api_key_index - self.api_key_index: + private_keys[self.api_key_index] = private_key + return private_keys + + def create_client(self, api_key_index=None): self.signer.CreateClient.argtypes = [ ctypes.c_char_p, ctypes.c_char_p, @@ -118,12 +215,13 @@ def create_client(self): ctypes.c_int, ctypes.c_longlong, ] + api_key_index = api_key_index or self.api_key_index self.signer.CreateClient.restype = ctypes.c_char_p err = self.signer.CreateClient( self.url.encode("utf-8"), - self.private_key.encode("utf-8"), + self.api_key_dict[api_key_index].encode("utf-8"), self.chain_id, - self.api_key_index, + api_key_index, self.account_index, ) @@ -141,7 +239,16 @@ def check_client(self): ] self.signer.CheckClient.restype = ctypes.c_char_p - result = self.signer.CheckClient(self.api_key_index, self.account_index) + for api_key in range(self.api_key_index, self.end_api_key_index + 1): + result = self.signer.CheckClient(api_key, self.account_index) + if result: + return result.decode("utf-8") + f" on api key {self.api_key_index}" + return result.decode("utf-8") if result else None + + def switch_api_key(self, api_key: int): + self.signer.SwitchAPIKey.argtypes = [ctypes.c_int] + self.signer.CheckClient.restype = ctypes.c_char_p + result = self.signer.SwitchAPIKey(api_key) return result.decode("utf-8") if result else None def create_api_key(self, seed=""): @@ -182,6 +289,16 @@ def sign_change_api_key(self, eth_private_key, new_pubkey: str, nonce: int): tx_info["L1Sig"] = signature.signature.to_0x_hex() return json.dumps(tx_info), None + def get_api_key_nonce(self, api_key_index: int, nonce: int) -> Tuple[int, int]: + if api_key_index != -1 and nonce != -1: + return api_key_index, nonce + if nonce != -1: + if self.api_key_index == self.end_api_key_index: + return self.nonce_manager.next_nonce() + else: + raise Exception("ambiguous api key") + return self.nonce_manager.next_nonce() + def sign_create_order( self, market_index, @@ -419,6 +536,7 @@ async def change_api_key(self, eth_private_key: str, new_pubkey: str, nonce=-1): logging.debug(f"Change Pub Key Send Tx Response: {api_response}") return api_response, None + @process_api_key_and_nonce async def create_order( self, market_index, @@ -432,6 +550,7 @@ async def create_order( trigger_price=NIL_TRIGGER_PRICE, order_expiry=-1, nonce=-1, + api_key_index=-1, ) -> (CreateOrder, TxHash, str): tx_info, error = self.sign_create_order( market_index, @@ -455,7 +574,15 @@ async def create_order( return CreateOrder.from_json(tx_info), api_response, None async def create_market_order( - self, market_index, client_order_index, base_amount, avg_execution_price, is_ask, reduce_only: bool = False + self, + market_index, + client_order_index, + base_amount, + avg_execution_price, + is_ask, + reduce_only: bool = False, + nonce=-1, + api_key_index=-1, ) -> (CreateOrder, TxHash, str): return await self.create_order( market_index, @@ -467,9 +594,12 @@ async def create_market_order( time_in_force=self.ORDER_TIME_IN_FORCE_IMMEDIATE_OR_CANCEL, order_expiry=self.DEFAULT_IOC_EXPIRY, reduce_only=reduce_only, + nonce=nonce, + api_key_index=api_key_index, ) - async def cancel_order(self, market_index, order_index, nonce=-1) -> (CancelOrder, TxHash, str): + @process_api_key_and_nonce + async def cancel_order(self, market_index, order_index, nonce=-1, api_key_index=-1) -> (CancelOrder, TxHash, str): tx_info, error = self.sign_cancel_order(market_index, order_index, nonce) if error is not None: return None, None, error @@ -479,7 +609,8 @@ async def cancel_order(self, market_index, order_index, nonce=-1) -> (CancelOrde logging.debug(f"Cancel Order Send Tx Response: {api_response}") return CancelOrder.from_json(tx_info), api_response, None - async def withdraw(self, usdc_amount, nonce=-1) -> (Withdraw, TxHash): + @process_api_key_and_nonce + async def withdraw(self, usdc_amount, nonce=-1, api_key_index=-1) -> (Withdraw, TxHash): usdc_amount = int(usdc_amount * self.USDC_TICKER_SCALE) tx_info, error = self.sign_withdraw(usdc_amount, nonce) @@ -499,9 +630,10 @@ async def create_sub_account(self, nonce=-1): api_response = await self.send_tx(tx_type=self.TX_TYPE_CREATE_SUB_ACCOUNT, tx_info=tx_info) logging.debug(f"Create Sub Account Send Tx Response: {api_response}") - return api_response, None + return tx_info, api_response, None - async def cancel_all_orders(self, time_in_force, time, nonce=-1): + @process_api_key_and_nonce + async def cancel_all_orders(self, time_in_force, time, nonce=-1, api_key_index=-1): tx_info, error = self.sign_cancel_all_orders(time_in_force, time, nonce) if error is not None: return None, error @@ -509,9 +641,12 @@ async def cancel_all_orders(self, time_in_force, time, nonce=-1): api_response = await self.send_tx(tx_type=self.TX_TYPE_CANCEL_ALL_ORDERS, tx_info=tx_info) logging.debug(f"Cancel All Orders Send Tx Response: {api_response}") - return api_response, None + return tx_info, api_response, None - async def modify_order(self, market_index, order_index, base_amount, price, trigger_price, nonce=-1): + @process_api_key_and_nonce + async def modify_order( + self, market_index, order_index, base_amount, price, trigger_price, nonce=-1, api_key_index=-1 + ): tx_info, error = self.sign_modify_order(market_index, order_index, base_amount, price, trigger_price, nonce) if error is not None: return None, error @@ -521,7 +656,8 @@ async def modify_order(self, market_index, order_index, base_amount, price, trig logging.debug(f"Modify Order Send Tx Response: {api_response}") return api_response, None - async def transfer(self, to_account_index, usdc_amount, nonce=-1): + @process_api_key_and_nonce + async def transfer(self, to_account_index, usdc_amount, nonce=-1, api_key_index=-1): usdc_amount = int(usdc_amount * self.USDC_TICKER_SCALE) tx_info, error = self.sign_transfer(to_account_index, usdc_amount, nonce) @@ -533,7 +669,10 @@ async def transfer(self, to_account_index, usdc_amount, nonce=-1): logging.debug(f"Transfer Send Tx Response: {api_response}") return api_response, None - async def create_public_pool(self, operator_fee, initial_total_shares, min_operator_share_rate, nonce=-1): + @process_api_key_and_nonce + async def create_public_pool( + self, operator_fee, initial_total_shares, min_operator_share_rate, nonce=-1, api_key_index=-1 + ): tx_info, error = self.sign_create_public_pool( operator_fee, initial_total_shares, min_operator_share_rate, nonce ) @@ -545,7 +684,10 @@ async def create_public_pool(self, operator_fee, initial_total_shares, min_opera logging.debug(f"Create Public Pool Send Tx Response: {api_response}") return api_response, None - async def update_public_pool(self, public_pool_index, status, operator_fee, min_operator_share_rate, nonce=-1): + @process_api_key_and_nonce + async def update_public_pool( + self, public_pool_index, status, operator_fee, min_operator_share_rate, nonce=-1, api_key_index=-1 + ): tx_info, error = self.sign_update_public_pool( public_pool_index, status, operator_fee, min_operator_share_rate, nonce ) @@ -557,7 +699,8 @@ async def update_public_pool(self, public_pool_index, status, operator_fee, min_ logging.debug(f"Update Public Pool Send Tx Response: {api_response}") return api_response, None - async def mint_shares(self, public_pool_index, share_amount, nonce=-1): + @process_api_key_and_nonce + async def mint_shares(self, public_pool_index, share_amount, nonce=-1, api_key_index=-1): tx_info, error = self.sign_mint_shares(public_pool_index, share_amount, nonce) if error is not None: return None, error @@ -567,7 +710,8 @@ async def mint_shares(self, public_pool_index, share_amount, nonce=-1): logging.debug(f"Mint Shares Send Tx Response: {api_response}") return api_response, None - async def burn_shares(self, public_pool_index, share_amount, nonce=-1): + @process_api_key_and_nonce + async def burn_shares(self, public_pool_index, share_amount, nonce=-1, api_key_index=-1): tx_info, error = self.sign_burn_shares(public_pool_index, share_amount, nonce) if error is not None: return None, error diff --git a/lighter/signers/.keep b/lighter/signers/.keep new file mode 100644 index 0000000..e69de29 diff --git a/lighter/signers/signer-amd64.so b/lighter/signers/signer-amd64.so index 85b688e..0ad5d7e 100644 Binary files a/lighter/signers/signer-amd64.so and b/lighter/signers/signer-amd64.so differ diff --git a/lighter/signers/signer-arm64.dylib b/lighter/signers/signer-arm64.dylib index bf526f7..ef2f76e 100644 Binary files a/lighter/signers/signer-arm64.dylib and b/lighter/signers/signer-arm64.dylib differ diff --git a/pyproject.toml b/pyproject.toml index 763a578..22ce64e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,6 +23,7 @@ pydantic = ">=2" typing-extensions = ">=4.7.1" websockets = ">= 12.0.0" eth-account = ">=0.13.4" +requests = ">=2.31.0" [tool.poetry.dev-dependencies] pytest = ">=7.2.1" diff --git a/requirements.txt b/requirements.txt index d06b125..a21c5cd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,4 +6,5 @@ typing-extensions >= 4.7.1 aiohttp >= 3.0.0 aiohttp-retry >= 2.8.3 websockets >= 12.0.0 -eth-account >= 0.13.4 \ No newline at end of file +eth-account >= 0.13.4 +requests >= 2.31.0 \ No newline at end of file diff --git a/setup.py b/setup.py index ea7313f..fa4e4f5 100644 --- a/setup.py +++ b/setup.py @@ -32,6 +32,7 @@ "typing-extensions >= 4.7.1", "websockets >= 12.0.0", "eth-account >= 0.13.4", + "requests >= 2.31.0", ] setup(