From 2bcb45a36815415890678ea7c1de8c25e25e3b1a Mon Sep 17 00:00:00 2001 From: belbix <7453635@gmail.com> Date: Mon, 20 Dec 2021 16:43:54 +0300 Subject: [PATCH 01/65] error status --- src/main/java/pro/belbix/ethparser/entity/ErrorEntity.java | 1 + src/main/java/pro/belbix/ethparser/service/ErrorService.java | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/pro/belbix/ethparser/entity/ErrorEntity.java b/src/main/java/pro/belbix/ethparser/entity/ErrorEntity.java index 91928856..86a9a1e4 100644 --- a/src/main/java/pro/belbix/ethparser/entity/ErrorEntity.java +++ b/src/main/java/pro/belbix/ethparser/entity/ErrorEntity.java @@ -25,4 +25,5 @@ public class ErrorEntity { private String json; private String errorClass; private String network; + private Integer status; } diff --git a/src/main/java/pro/belbix/ethparser/service/ErrorService.java b/src/main/java/pro/belbix/ethparser/service/ErrorService.java index 5a0a8a53..5bbbffd9 100644 --- a/src/main/java/pro/belbix/ethparser/service/ErrorService.java +++ b/src/main/java/pro/belbix/ethparser/service/ErrorService.java @@ -68,6 +68,9 @@ public void startFixErrorService() { log.info("Load errors from db: " + listErrors.size()); for (ErrorEntity errorEntity : listErrors) { try { + if (errorEntity.getStatus() == 1) { + continue; + } parseObject(errorEntity); errorDbService.delete(errorEntity); } catch (Exception e) { @@ -80,7 +83,7 @@ public void startFixErrorService() { public void parseObject(ErrorEntity errorEntity) { if (errorEntity == null || errorEntity.getErrorClass() == null) { - throw new IllegalStateException("Detected unknown errorClass: " + errorEntity.toString()); + throw new IllegalStateException("Detected unknown errorClass: " + errorEntity); } switch (errorEntity.getErrorClass()) { case "DeployerTransactionsParser": From 17dc5c78ce0ccf95665f12a533ba7590508acdfe Mon Sep 17 00:00:00 2001 From: belbix <7453635@gmail.com> Date: Mon, 20 Dec 2021 16:46:12 +0300 Subject: [PATCH 02/65] error status2 --- src/main/java/pro/belbix/ethparser/service/ErrorService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/pro/belbix/ethparser/service/ErrorService.java b/src/main/java/pro/belbix/ethparser/service/ErrorService.java index 5bbbffd9..13b885c2 100644 --- a/src/main/java/pro/belbix/ethparser/service/ErrorService.java +++ b/src/main/java/pro/belbix/ethparser/service/ErrorService.java @@ -68,7 +68,7 @@ public void startFixErrorService() { log.info("Load errors from db: " + listErrors.size()); for (ErrorEntity errorEntity : listErrors) { try { - if (errorEntity.getStatus() == 1) { + if (errorEntity.getStatus() != null && errorEntity.getStatus() == 1) { continue; } parseObject(errorEntity); From 619677cc88e3d7bcccb40dd16d43ba6bc99fc907 Mon Sep 17 00:00:00 2001 From: belbix <7453635@gmail.com> Date: Mon, 20 Dec 2021 18:10:39 +0300 Subject: [PATCH 03/65] fix npe --- .../ethparser/web3/harvest/db/VaultActionsDBService.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/main/java/pro/belbix/ethparser/web3/harvest/db/VaultActionsDBService.java b/src/main/java/pro/belbix/ethparser/web3/harvest/db/VaultActionsDBService.java index 2ce75173..2ed12e36 100644 --- a/src/main/java/pro/belbix/ethparser/web3/harvest/db/VaultActionsDBService.java +++ b/src/main/java/pro/belbix/ethparser/web3/harvest/db/VaultActionsDBService.java @@ -5,7 +5,6 @@ import static pro.belbix.ethparser.service.AbiProviderService.BSC_NETWORK; import static pro.belbix.ethparser.service.AbiProviderService.ETH_NETWORK; import static pro.belbix.ethparser.service.AbiProviderService.MATIC_NETWORK; -import static pro.belbix.ethparser.web3.contracts.ContractConstants.MATIC_BLOCK_NUMBER_06_JUL_2021; import static pro.belbix.ethparser.web3.contracts.ContractConstants.iPS_ADDRESS; import com.fasterxml.jackson.databind.ObjectMapper; @@ -31,6 +30,7 @@ @Service @Log4j2 public class VaultActionsDBService { + private final static ObjectMapper objectMapper = new ObjectMapper(); private final HarvestRepository harvestRepository; @@ -195,6 +195,9 @@ private double calculateActualTvl(HarvestDTO dto, Double farmPrice) { } catch (Exception ignored) { } if (tvl == 0.0) { + if (dto.getLastUsdTvl() == null) { + return 0; + } return dto.getLastUsdTvl(); } if (Double.isInfinite(tvl) || Double.isNaN(tvl)) { @@ -212,8 +215,7 @@ public BigInteger lastBlock(String network) { return BigInteger.valueOf(5993570L); } else if (MATIC_NETWORK.equals(network)) { return BigInteger.valueOf(16566542L); - } - else { + } else { return new BigInteger("0"); } } From 5915a7ca38bb1b225948888c82f83405cf3ef5c6 Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 22 Dec 2021 16:36:16 +0200 Subject: [PATCH 04/65] add calculation profit a.k.a yield (wip) --- .../controllers/ProfitController.java | 33 +++++ .../repositories/v0/HarvestRepository.java | 36 +++++ .../ethparser/service/ProfitService.java | 130 ++++++++++++++++++ .../controllers/ProfitControllerTest.java | 40 ++++++ src/test/resources/sql/1_eth_contracts.sql | 9 +- src/test/resources/sql/3_eth_vaults.sql | 4 +- src/test/resources/sql/4_eth_pools.sql | 3 +- src/test/resources/sql/harvest.sql | 3 + 8 files changed, 255 insertions(+), 3 deletions(-) create mode 100644 src/main/java/pro/belbix/ethparser/controllers/ProfitController.java create mode 100644 src/main/java/pro/belbix/ethparser/service/ProfitService.java create mode 100644 src/test/java/pro/belbix/ethparser/controllers/ProfitControllerTest.java diff --git a/src/main/java/pro/belbix/ethparser/controllers/ProfitController.java b/src/main/java/pro/belbix/ethparser/controllers/ProfitController.java new file mode 100644 index 00000000..54c891d5 --- /dev/null +++ b/src/main/java/pro/belbix/ethparser/controllers/ProfitController.java @@ -0,0 +1,33 @@ +package pro.belbix.ethparser.controllers; + +import io.swagger.v3.oas.annotations.Parameter; +import lombok.extern.log4j.Log4j2; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; +import pro.belbix.ethparser.service.ProfitService; + +@RestController +@Log4j2 +public class ProfitController { + + private final ProfitService profitService; + + public ProfitController(ProfitService profitService){ + + this.profitService = profitService; + } + + @RequestMapping(value = "api/profit/calculation", method = RequestMethod.GET) + public Double fetchProfit( + @RequestParam("address") @Parameter(description = "Vault address") String address, + @RequestParam(value = "start", required = false, defaultValue = "0") + @Parameter(description = "Block creation time from") String start, + @RequestParam(value = "end", required = false, defaultValue = Long.MAX_VALUE + "") + @Parameter(description = "Block creation time to") String end + ) { + + return profitService.calculationProfitForPeriod(address, start, end); + } +} diff --git a/src/main/java/pro/belbix/ethparser/repositories/v0/HarvestRepository.java b/src/main/java/pro/belbix/ethparser/repositories/v0/HarvestRepository.java index 2e818e8f..b45c4ed6 100644 --- a/src/main/java/pro/belbix/ethparser/repositories/v0/HarvestRepository.java +++ b/src/main/java/pro/belbix/ethparser/repositories/v0/HarvestRepository.java @@ -100,6 +100,32 @@ Integer fetchAllPoolsUsersQuantity( @Param("network") String network ); + @Query(nativeQuery = true, value = "" + + " select coalesce(sum(t.deposit), 0) as difference " + + " from ( " + + " select - sum(harvest_tx.owner_balance) as deposit " + + " from harvest_tx " + + " where harvest_tx.owner = :owner " + + " and harvest_tx.vault = :vault " + + " and harvest_tx.network = :network " + + " and harvest_tx.method_name = 'Deposit' " + + " and harvest_tx.block_date <= :block_date " + + " union all " + + " select sum(harvest_tx.owner_balance) as withdrow " + + " from harvest_tx " + + " where harvest_tx.owner = :owner " + + " and harvest_tx.vault = :vault " + + " and harvest_tx.network = :network " + + " and harvest_tx.method_name = 'Withdraw' " + + " and harvest_tx.block_date <= :block_date " + + " ) t ") + Double fetchTotalDifferenceDepositAndWithdraw( + @Param("vault") String vault, + @Param("owner") String owner, + @Param("block_date") long blockDate, + @Param("network") String network + ); + HarvestDTO findFirstByNetworkOrderByBlockDesc(String network); @Query("select max(t.blockDate) - min(t.blockDate) as period from HarvestDTO t where " @@ -134,6 +160,16 @@ List fetchAllByOwner( @Param("network") String network ); + @Query("select t from HarvestDTO t where " + + "t.owner = :owner " + + "and t.blockDate between :from and :to " + + "order by t.blockDate asc") + List fetchAllByOwnerWithoutNetwork( + @Param("owner") String owner, + @Param("from") long from, + @Param("to") long to + ); + @Query("select t from HarvestDTO t where " + "(t.ownerBalance is null " + "or t.ownerBalanceUsd is null) " diff --git a/src/main/java/pro/belbix/ethparser/service/ProfitService.java b/src/main/java/pro/belbix/ethparser/service/ProfitService.java new file mode 100644 index 00000000..199ccd4b --- /dev/null +++ b/src/main/java/pro/belbix/ethparser/service/ProfitService.java @@ -0,0 +1,130 @@ +package pro.belbix.ethparser.service; + +import static pro.belbix.ethparser.utils.CommonUtils.parseLong; +import static pro.belbix.ethparser.web3.abi.FunctionsNames.BALANCE_OF; +import static pro.belbix.ethparser.web3.abi.FunctionsNames.GET_PRICE_PER_FULL_SHARE; + +import java.math.BigInteger; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import lombok.extern.log4j.Log4j2; +import org.springframework.stereotype.Service; +import pro.belbix.ethparser.dto.v0.HarvestDTO; +import pro.belbix.ethparser.repositories.v0.HarvestRepository; +import pro.belbix.ethparser.web3.EthBlockService; +import pro.belbix.ethparser.web3.abi.FunctionsUtils; +import pro.belbix.ethparser.web3.contracts.db.ContractDbService; + +@Service +@Log4j2 +public class ProfitService { + + private final HarvestRepository harvestRepository; + private final ContractDbService contractDbService; + private final EthBlockService ethBlockService; + private final FunctionsUtils functionsUtils; + + + public ProfitService(HarvestRepository harvestRepository, + ContractDbService contractDbService, + EthBlockService ethBlockService, + FunctionsUtils functionsUtils + ) { + this.harvestRepository = harvestRepository; + this.contractDbService = contractDbService; + this.ethBlockService = ethBlockService; + this.functionsUtils = functionsUtils; + + } + + + public Double calculationProfitForPeriod(String address, String start, String end) { + + List harvestDTOS = harvestRepository.fetchAllByOwnerWithoutNetwork( + address.toLowerCase(), + parseLong(start, 0), + parseLong(end, Long.MAX_VALUE)); + + return calculationTotalYield(harvestDTOS.stream() + .collect(Collectors.groupingBy(HarvestDTO::getVault, Collectors.toList())), + parseLong(start, 0), parseLong(end, Long.MAX_VALUE)); + } + + private Double calculationTotalYield(Map> maps, + Long start, Long end) { + + return maps.values().stream().map(e -> calcYieldByVault(e, start, end)) + .reduce(0.0, Double::sum); + } + + + private Double calcYieldByVault(List v, Long start, Long end) { + long startBlockNumber = (v.get(0).getBlock() * start) / v.get(0).getBlockDate(); +// long endBlockNumber = (v.get(v.size() - 1).getBlock() * end) / v.get(v.size() - 1).getBlockDate();// random block number (wip) + Long endBlockNumber = ethBlockService.getLastBlock(v.get(0).getNetwork()); + + String vaultAddress = v.get(0).getVaultAddress(); + String network = v.get(0).getNetwork(); + String owner = v.get(0).getOwner(); + String vault = v.get(0).getVault(); + + String poolAddress = contractDbService.getPoolContractByVaultAddress( + vaultAddress, startBlockNumber, network) + .orElseThrow().getAddress(); + + return getYieldValue(owner, vault, vaultAddress, network, startBlockNumber, endBlockNumber, + poolAddress, start, end); + } + + private double getYieldValue(String owner, String vault, String vaultAddress, String network, + long startBlockNumber, Long endBlockNumber, String poolAddress, Long start, Long end) { + + double underlyingBalanceStart = getUnderlyingBalance(owner, startBlockNumber, vaultAddress, + network, poolAddress); + + double underlyingBalanceEnd = getUnderlyingBalance(owner, endBlockNumber, vaultAddress, + network, poolAddress); + + double totalDifferenceDepositAndWithdrawUnderlyingStart = + harvestRepository.fetchTotalDifferenceDepositAndWithdraw( + vault, owner, start, network); + + double totalDifferenceDepositAndWithdrawUnderlyingEnd = + harvestRepository.fetchTotalDifferenceDepositAndWithdraw( + vault, owner, end, network); + + double yieldEnd = underlyingBalanceEnd + totalDifferenceDepositAndWithdrawUnderlyingEnd; + double yieldStart = underlyingBalanceStart + totalDifferenceDepositAndWithdrawUnderlyingStart; + + return yieldEnd - yieldStart; + } + + private double getUnderlyingBalance(String owner, long block, + String vaultAddress, String network, String poolAddress) { + return functionsUtils. + parseAmount(getCoinBalance(block, owner, vaultAddress, network) + .add(getCoinBalance(block, owner, poolAddress, network)), + vaultAddress, network) + * pricePerFullShare(vaultAddress, block, network); + } + + private double pricePerFullShare(String vaultAddress, Long block, String network) { + var sharePriceInt = functionsUtils.callIntByName( + GET_PRICE_PER_FULL_SHARE, vaultAddress, block, network) + .orElse(BigInteger.ZERO); + + if (BigInteger.ONE.equals(sharePriceInt)) { + return 0.0; + } + return functionsUtils + .parseAmount(sharePriceInt, vaultAddress, network); + + } + + private BigInteger getCoinBalance(long block, String owner, String address, String network) { + return functionsUtils.callIntByNameWithAddressArg(BALANCE_OF, owner, address, block, + network) + .orElseThrow(() -> new IllegalStateException("Error get amount from " + address)); + } +} diff --git a/src/test/java/pro/belbix/ethparser/controllers/ProfitControllerTest.java b/src/test/java/pro/belbix/ethparser/controllers/ProfitControllerTest.java new file mode 100644 index 00000000..183b2cc3 --- /dev/null +++ b/src/test/java/pro/belbix/ethparser/controllers/ProfitControllerTest.java @@ -0,0 +1,40 @@ +package pro.belbix.ethparser.controllers; + +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.web.servlet.MockMvc; +import pro.belbix.ethparser.Application; +import pro.belbix.ethparser.repositories.v0.HarvestRepository; + +@SpringBootTest(classes = Application.class) +@ContextConfiguration +@AutoConfigureMockMvc +public class ProfitControllerTest { + + @Autowired + private MockMvc mockMvc; + + @Autowired + private HarvestRepository harvestRepository; + + + @ParameterizedTest(name = "{index} => address={0}, start={1}, end={2}") + @CsvSource({ + "0x858128d2f83dbb226b6cf29bffe5e7e129c3a128, 1613588289, 1615434501", + "0xf1b85414473eccd659deeacee347bbc17171dd32, 1613588289, 1640168651" + }) + public void testCalculation(String address, String start, String end) throws Exception { + + this.mockMvc.perform( + get("/api/profit/calculation?address=" + address + "&start=" + start + "&end=" + end)) + .andExpect(status().isOk()); + } + +} diff --git a/src/test/resources/sql/1_eth_contracts.sql b/src/test/resources/sql/1_eth_contracts.sql index e1cf11bd..3f90c516 100644 --- a/src/test/resources/sql/1_eth_contracts.sql +++ b/src/test/resources/sql/1_eth_contracts.sql @@ -1140,4 +1140,11 @@ INSERT INTO public.eth_contracts (id, address, created, name, network, type, un INSERT INTO public.eth_contracts (id, address, created, name, network, type, underlying, updated, created_date, updated_date) VALUES (10067, '0xe506e7c500fe6e094c37682dd7ed847306d0bccb', 16572694, 'UNI_LP_WBTC_miFARM', 'matic', 2, null, 16645038, 1625597732, 1625761021); INSERT INTO public.eth_contracts (id, address, created, name, network, type, underlying, updated, created_date, updated_date) VALUES (10068, '0x1bfd67037b42cf73acf2047067bd4f2c47d9bfd6', 16572694, 'WBTC', 'matic', 4, null, 16645038, 1625597732, 1625761021); INSERT INTO public.eth_contracts (id, address, created, name, network, type, underlying, updated, created_date, updated_date) VALUES (10069, '0xf6a637525402643b0654a54bead2cb9a83c8b498', 16572694, 'UNI_LP_WBTC_USDC', 'matic', 2, null, 16645038, 1625597732, 1625761021); -INSERT INTO public.eth_contracts (id, address, created, name, network, type, underlying, updated, created_date, updated_date) VALUES (10109, '0x0392f36d2896c966e141c8fd9eca58a7ca9fa8af', 16657130, 'S_UNKNOWN_NAME_#V1', 'matic', 5, null, null, 1625795083, null); \ No newline at end of file +INSERT INTO public.eth_contracts (id, address, created, name, network, type, underlying, updated, created_date, updated_date) VALUES (10109, '0x0392f36d2896c966e141c8fd9eca58a7ca9fa8af', 16657130, 'S_UNKNOWN_NAME_#V1', 'matic', 5, null, null, 1625795083, null); +INSERT INTO public.eth_contracts (id, address, created, name, network, type, underlying, updated, created_date, updated_date) VALUES (10443, '0xe7c9d242137896741b70cefef701bbb4dcb158ec', 21718484, 'V_USDC_#V2', 'matic', 0, null, 22597799, 1637697287, 1639671802); +INSERT INTO public.eth_contracts (id, address, created, name, network, type, underlying, updated, created_date, updated_date) VALUES (10445, '0xea4eca0fbeb0fcaae61d2fa2b2877d435faeee1c', 21718504, 'P_USDC', 'matic', 1, null, 21985528, 1637697327, 1638302127); +INSERT INTO public.eth_contracts (id, address, created, name, network, type, underlying, updated, created_date, updated_date) VALUES (10452, '0xebafc813f66c3142e7993a88ee3361a1f4bdab16', 0, 'CONTROLLER_#V2', 'matic', 3, null, null, 1590824836, null); +INSERT INTO public.eth_contracts (id, address, created, name, network, type, underlying, updated, created_date, updated_date) VALUES (10402, '0x7f316d2498a234670554d1a890a364b0bd9d88e2', 20625022, 'V_KyberDMMLPUSDC-jCHF_USDC_jCHF_#V1', 'matic', 0, null, 22250517, 1635236145, 1638903565); +INSERT INTO public.eth_contracts (id, address, created, name, network, type, underlying, updated, created_date, updated_date) VALUES (10444, '0x08043b010ad2cd72db75878201fde4b03e28e069', 21718489, 'S_USDC_#V2', 'matic', 5, null, null, 1637697297, null); +INSERT INTO public.eth_contracts (id, address, created, name, network, type, underlying, updated, created_date, updated_date) VALUES (10446, '0x9048a123ae6c86a2b9fdc3ea28ed1911de8363a6', 21718513, 'V_WETH', 'matic', 0, null, 22598271, 1637697345, 1639672774); +INSERT INTO public.eth_contracts (id, address, created, name, network, type, underlying, updated, created_date, updated_date) VALUES (10447, '0x7ad709df8b897b3b623c5413e92a6f70ca563e1d', 21718518, 'S_WETH', 'matic', 5, null, null, 1637697355, null); diff --git a/src/test/resources/sql/3_eth_vaults.sql b/src/test/resources/sql/3_eth_vaults.sql index b96c51c6..2184458f 100644 --- a/src/test/resources/sql/3_eth_vaults.sql +++ b/src/test/resources/sql/3_eth_vaults.sql @@ -151,4 +151,6 @@ INSERT INTO public.eth_vaults (id, decimals, name, symbol, underlying_unit, upda INSERT INTO public.eth_vaults (id, decimals, name, symbol, underlying_unit, updated_block, contract, controller_id, governance_id, strategy_id, underlying_id) VALUES (10006, 18, 'miFARM_UNI-V2', 'bfUNI-V2', 1000000000000000000, 16644916, 10070, 10071, 10058, null, 10072); INSERT INTO public.eth_vaults (id, decimals, name, symbol, underlying_unit, updated_block, contract, controller_id, governance_id, strategy_id, underlying_id) VALUES (10007, 18, 'miFARM_UNI-V2', 'bfUNI-V2', 1000000000000000000, 16644946, 10077, 10071, 10058, null, 10059); INSERT INTO public.eth_vaults (id, decimals, name, symbol, underlying_unit, updated_block, contract, controller_id, governance_id, strategy_id, underlying_id) VALUES (10039, 18, 'miFARM_SLP', 'bfSLP', 1000000000000000000, 16644978, 10103, 10071, 10058, null, 10104); -INSERT INTO public.eth_vaults (id, decimals, name, symbol, underlying_unit, updated_block, contract, controller_id, governance_id, strategy_id, underlying_id) VALUES (10040, 18, 'miFARM_SLP', 'bfSLP', 1000000000000000000, 16657148, 10105, 10071, 10058, 10107, 10104); \ No newline at end of file +INSERT INTO public.eth_vaults (id, decimals, name, symbol, underlying_unit, updated_block, contract, controller_id, governance_id, strategy_id, underlying_id) VALUES (10040, 18, 'miFARM_SLP', 'bfSLP', 1000000000000000000, 16657148, 10105, 10071, 10058, 10107, 10104); +INSERT INTO public.eth_vaults (id, decimals, name, symbol, underlying_unit, updated_block, contract, controller_id, governance_id, strategy_id, underlying_id) VALUES (10050, 6, 'miFARM_USDC', 'bfUSDC', 1000000, 22597799, 10443, 10452, 10058, 10444, 10063); +INSERT INTO public.eth_vaults (id, decimals, name, symbol, underlying_unit, updated_block, contract, controller_id, governance_id, strategy_id, underlying_id) VALUES (10051, 18, 'miFARM_WETH', 'bfWETH', 1000000000000000000, 22598271, 10446, 10452, 10058, 10447, 10060); diff --git a/src/test/resources/sql/4_eth_pools.sql b/src/test/resources/sql/4_eth_pools.sql index 122505fd..d9772d4d 100644 --- a/src/test/resources/sql/4_eth_pools.sql +++ b/src/test/resources/sql/4_eth_pools.sql @@ -190,4 +190,5 @@ INSERT INTO public.eth_pools (id, updated_block, contract, controller, governanc INSERT INTO public.eth_pools (id, updated_block, contract, controller, governance, lp_token, owner, reward_token) VALUES (10005, 16572694, 10065, 10057, 10058, 10056, 10058, 10066); INSERT INTO public.eth_pools (id, updated_block, contract, controller, governance, lp_token, owner, reward_token) VALUES (10006, 16644936, 10076, 10071, 10058, 10070, 10058, 10066); INSERT INTO public.eth_pools (id, updated_block, contract, controller, governance, lp_token, owner, reward_token) VALUES (10007, 16644965, 10079, 10071, 10058, 10077, 10058, 10066); -INSERT INTO public.eth_pools (id, updated_block, contract, controller, governance, lp_token, owner, reward_token) VALUES (10039, 16645038, 10108, 10071, 10058, 10105, 10058, 10066); \ No newline at end of file +INSERT INTO public.eth_pools (id, updated_block, contract, controller, governance, lp_token, owner, reward_token) VALUES (10039, 16645038, 10108, 10071, 10058, 10105, 10058, 10066); +INSERT INTO public.eth_pools (id, updated_block, contract, controller, governance, lp_token, owner, reward_token) VALUES (10077, 21985528, 10445, 10452, 10058, 10443, 10058, 10066); diff --git a/src/test/resources/sql/harvest.sql b/src/test/resources/sql/harvest.sql index 06f73cd9..de3ab105 100644 --- a/src/test/resources/sql/harvest.sql +++ b/src/test/resources/sql/harvest.sql @@ -93,3 +93,6 @@ VALUES ('0x3457c2b58d728db60d8d4e9fa4931d1819947614917983e5a8e7e9a215095216_154' INSERT INTO public.harvest_tx (id, all_owners_count, all_pools_owners_count, amount, amount_in, block, block_date, confirmed, hash, last_all_usd_tvl, last_gas, last_tvl, last_usd_tvl, lp_stat, method_name, migrated, owner, owner_balance, owner_balance_usd, owner_count, prices, profit, profit_usd, share_price, total_amount, underlying_price, usd_amount, vault, network, vault_address, underlying_address) VALUES ('0x46571abfbf3368e27318f42e14e26384a71f986806fd65daca9b213e3b077d2a_242', 5, 3, 0.000000031272302486, null, 18493981, 1630172828, 1, '0x46571abfbf3368e27318f42e14e26384a71f986806fd65daca9b213e3b077d2a', null, 6.326405532, 0.004576514740784796, 579217, '{"coin1":"USDC","coin1Address":"0x2791bca1f2de4661ed88a30c99a7a9449aa84174","coin2":"WETH","coin2Address":"0x7ceb23fd6bc0add59e62ac25578270cff1b9f619","amount1":289528.1661267551,"amount2":89.13372847530734,"price1":1.0,"price2":3250.052845311043}', 'Deposit', false, '0x5874e4ea2f891ad337faee094363bf5d7afcc61a', 0.000000031613611839, 4.001113961586612, 0, null, null, null, 1.0109141101279588, null, null, 4, 'V_SUSHI_USDC_WETH', 'matic', '0xf76a0c5083b895c76ecbf30121f036849137d545', null); INSERT INTO public.harvest_tx (id, all_owners_count, all_pools_owners_count, amount, amount_in, block, block_date, confirmed, hash, last_all_usd_tvl, last_gas, last_tvl, last_usd_tvl, lp_stat, method_name, migrated, owner, owner_balance, owner_balance_usd, owner_count, prices, profit, profit_usd, share_price, total_amount, underlying_price, usd_amount, vault, network, vault_address, underlying_address) VALUES ('0xb140f7e195b4e4a75a5b97d8a665c22511bf01de20a09ee3d72a3c4ba15d5b0e_778', 6, 3, 0.000000000000015633, null, 18494511, 1630173920, 1, '0xb140f7e195b4e4a75a5b97d8a665c22511bf01de20a09ee3d72a3c4ba15d5b0e', null, 7.02231014, 0.004576514740800599, 579052, '{"coin1":"USDC","coin1Address":"0x2791bca1f2de4661ed88a30c99a7a9449aa84174","coin2":"WETH","coin2Address":"0x7ceb23fd6bc0add59e62ac25578270cff1b9f619","amount1":289578.6683028402,"amount2":89.11838503490969,"price1":1.0,"price2":3248.19017054529}', 'Deposit', false, '0x747276019e3340104c96397bf6537ad01f93d7df', 0.000000000000015803, 0.0000019995042817414836, 0, null, null, null, 1.0109141101279588, null, null, 0, 'V_SUSHI_USDC_WETH', 'matic', '0xf76a0c5083b895c76ecbf30121f036849137d545', null); +INSERT INTO public.harvest_tx (id, all_owners_count, all_pools_owners_count, amount, amount_in, block, block_date, confirmed, hash, last_all_usd_tvl, last_gas, last_tvl, last_usd_tvl, lp_stat, method_name, migrated, owner, owner_balance, owner_balance_usd, owner_count, prices, profit, profit_usd, share_price, total_amount, underlying_price, usd_amount, vault, network, vault_address, underlying_address) VALUES ('0x625a3ee376fb2d2eba5db5f23364594cd87a09f35a6510e5f2274ce48a897db3_767', 1943, 156, 0.00000002465398013, null, 22478088, 1639403843, 1, '0x625a3ee376fb2d2eba5db5f23364594cd87a09f35a6510e5f2274ce48a897db3', null, 30, 0.0007538475609659471, 111093, '{"coin1":"USDC","coin1Address":"0x2791bca1f2de4661ed88a30c99a7a9449aa84174","coin2":"WETH","coin2Address":"0x7ceb23fd6bc0add59e62ac25578270cff1b9f619","amount1":55515.9914631261,"amount2":13.960744455081848,"price1":1.0,"price2":3980.9778353221886}', 'Deposit', false, '0xf1b85414473eccd659deeacee347bbc17171dd32', 0.000000025799754059, 3.8020717889594717, 23, null, null, null, 1.0464741970020397, null, null, 4, 'V_SUSHI_USDC_WETH_#V1', 'matic', '0xf76a0c5083b895c76ecbf30121f036849137d545', null); +INSERT INTO public.harvest_tx (id, all_owners_count, all_pools_owners_count, amount, amount_in, block, block_date, confirmed, hash, last_all_usd_tvl, last_gas, last_tvl, last_usd_tvl, lp_stat, method_name, migrated, owner, owner_balance, owner_balance_usd, owner_count, prices, profit, profit_usd, share_price, total_amount, underlying_price, usd_amount, vault, network, vault_address, underlying_address) VALUES ('0x4e4149e769317c169f3e8e5c3b28e3ec4818c3d4e8010f79c0ff939236d5c849_203', 2006, 161, 0.998017, null, 22636874, 1639756701, 1, '0x4e4149e769317c169f3e8e5c3b28e3ec4818c3d4e8010f79c0ff939236d5c849', null, 30, 531408.6788976708, 531409, null, 'Deposit', false, '0xf1b85414473eccd659deeacee347bbc17171dd32', 0.999999, 0.999999, 38, null, null, null, 1.001986, null, 1, 0, 'V_USDC_#V2', 'matic', '0xe7c9d242137896741b70cefef701bbb4dcb158ec', '0x2791bca1f2de4661ed88a30c99a7a9449aa84174'); +INSERT INTO public.harvest_tx (id, all_owners_count, all_pools_owners_count, amount, amount_in, block, block_date, confirmed, hash, last_all_usd_tvl, last_gas, last_tvl, last_usd_tvl, lp_stat, method_name, migrated, owner, owner_balance, owner_balance_usd, owner_count, prices, profit, profit_usd, share_price, total_amount, underlying_price, usd_amount, vault, network, vault_address, underlying_address) VALUES ('0x27201914f0e4f107709684b57f9e60e660207fbf97e0b748ba9f2e9d05c610af_655', 2024, 166, 0.498892, null, 22750126, 1640005921, 1, '0x27201914f0e4f107709684b57f9e60e660207fbf97e0b748ba9f2e9d05c610af', null, 1.5972011, 979375.2934887275, 979375, null, 'Deposit', false, '0xf1b85414473eccd659deeacee347bbc17171dd32', 0.499999, 0.499999, 43, null, null, null, 1.002219, null, 1, 0, 'V_USDC_#V2', 'matic', '0xe7c9d242137896741b70cefef701bbb4dcb158ec', '0x2791bca1f2de4661ed88a30c99a7a9449aa84174'); From d89818086ccca620579d048df9a0cf71fb9ffe4e Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 23 Dec 2021 11:09:44 +0200 Subject: [PATCH 05/65] add new endpoint --- .../controllers/ProfitController.java | 19 +++++++++++++++++-- .../repositories/v0/HarvestRepository.java | 12 ++++++++++++ .../ethparser/service/ProfitService.java | 13 +++++++++++++ .../controllers/ProfitControllerTest.java | 15 ++++++++++++++- 4 files changed, 56 insertions(+), 3 deletions(-) diff --git a/src/main/java/pro/belbix/ethparser/controllers/ProfitController.java b/src/main/java/pro/belbix/ethparser/controllers/ProfitController.java index 54c891d5..f412c2b8 100644 --- a/src/main/java/pro/belbix/ethparser/controllers/ProfitController.java +++ b/src/main/java/pro/belbix/ethparser/controllers/ProfitController.java @@ -1,5 +1,7 @@ package pro.belbix.ethparser.controllers; +import static pro.belbix.ethparser.service.AbiProviderService.ETH_NETWORK; + import io.swagger.v3.oas.annotations.Parameter; import lombok.extern.log4j.Log4j2; import org.springframework.web.bind.annotation.RequestMapping; @@ -19,9 +21,9 @@ public ProfitController(ProfitService profitService){ this.profitService = profitService; } - @RequestMapping(value = "api/profit/calculation", method = RequestMethod.GET) + @RequestMapping(value = "api/profit/total", method = RequestMethod.GET) public Double fetchProfit( - @RequestParam("address") @Parameter(description = "Vault address") String address, + @RequestParam("address") @Parameter(description = "Owner address") String address, @RequestParam(value = "start", required = false, defaultValue = "0") @Parameter(description = "Block creation time from") String start, @RequestParam(value = "end", required = false, defaultValue = Long.MAX_VALUE + "") @@ -30,4 +32,17 @@ public Double fetchProfit( return profitService.calculationProfitForPeriod(address, start, end); } + + @RequestMapping(value = "api/profit/vault", method = RequestMethod.GET) + public Double fetchProfitByVault( + @RequestParam("address") @Parameter(description = "Vault address") String address, + @RequestParam(value = "network", required = false, defaultValue = ETH_NETWORK) String network, + @RequestParam(value = "start", required = false, defaultValue = "0") + @Parameter(description = "Block creation time from") String start, + @RequestParam(value = "end", required = false, defaultValue = Long.MAX_VALUE + "") + @Parameter(description = "Block creation time to") String end + ) { + + return profitService.calculationProfitByVaultForPeriod(address, network, start, end); + } } diff --git a/src/main/java/pro/belbix/ethparser/repositories/v0/HarvestRepository.java b/src/main/java/pro/belbix/ethparser/repositories/v0/HarvestRepository.java index b45c4ed6..45d3581b 100644 --- a/src/main/java/pro/belbix/ethparser/repositories/v0/HarvestRepository.java +++ b/src/main/java/pro/belbix/ethparser/repositories/v0/HarvestRepository.java @@ -170,6 +170,18 @@ List fetchAllByOwnerWithoutNetwork( @Param("to") long to ); + @Query("select t from HarvestDTO t where " + + "lower(t.vaultAddress) = lower(:vault) " + + "and t.blockDate between :from and :to " + + "and t.network = :network " + + "order by t.blockDate asc") + List fetchAllByVaultAddressAndNetwork( + @Param("vault") String vaultAddress, + @Param("from") long from, + @Param("to") long to, + @Param("network") String network + ); + @Query("select t from HarvestDTO t where " + "(t.ownerBalance is null " + "or t.ownerBalanceUsd is null) " diff --git a/src/main/java/pro/belbix/ethparser/service/ProfitService.java b/src/main/java/pro/belbix/ethparser/service/ProfitService.java index 199ccd4b..9658c4b7 100644 --- a/src/main/java/pro/belbix/ethparser/service/ProfitService.java +++ b/src/main/java/pro/belbix/ethparser/service/ProfitService.java @@ -127,4 +127,17 @@ private BigInteger getCoinBalance(long block, String owner, String address, Stri network) .orElseThrow(() -> new IllegalStateException("Error get amount from " + address)); } + + public Double calculationProfitByVaultForPeriod(String address, String network, String start, + String end) { + List harvestDTOS = harvestRepository.fetchAllByVaultAddressAndNetwork( + address.toLowerCase(), + parseLong(start, 0), + parseLong(end, Long.MAX_VALUE), + network); + + return calculationTotalYield(harvestDTOS.stream() + .collect(Collectors.groupingBy(HarvestDTO::getVault, Collectors.toList())), + parseLong(start, 0), parseLong(end, Long.MAX_VALUE)); + } } diff --git a/src/test/java/pro/belbix/ethparser/controllers/ProfitControllerTest.java b/src/test/java/pro/belbix/ethparser/controllers/ProfitControllerTest.java index 183b2cc3..99dd9d00 100644 --- a/src/test/java/pro/belbix/ethparser/controllers/ProfitControllerTest.java +++ b/src/test/java/pro/belbix/ethparser/controllers/ProfitControllerTest.java @@ -33,7 +33,20 @@ public class ProfitControllerTest { public void testCalculation(String address, String start, String end) throws Exception { this.mockMvc.perform( - get("/api/profit/calculation?address=" + address + "&start=" + start + "&end=" + end)) + get("/api/profit/total?address=" + address + "&start=" + start + "&end=" + end)) + .andExpect(status().isOk()); + } + + @ParameterizedTest(name = "{index} => address={0}, start={1}, end={2}") + @CsvSource({ + "0xe7c9d242137896741b70cefef701bbb4dcb158ec, matic, 1613588289, 1640168651" + }) + public void testCalculationByVault(String address, String network, String start, String end) + throws Exception { + + this.mockMvc.perform( + get("/api/profit/vault?address=" + address + + "&network=" + network + "&start=" + start + "&end=" + end)) .andExpect(status().isOk()); } From 2a65252a1b8a7376fd6f5abc6ea165c8e5bb18b5 Mon Sep 17 00:00:00 2001 From: Alex Date: Mon, 27 Dec 2021 12:14:34 +0200 Subject: [PATCH 06/65] fix null, add optional parameter end --- .../ethparser/service/ProfitService.java | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/main/java/pro/belbix/ethparser/service/ProfitService.java b/src/main/java/pro/belbix/ethparser/service/ProfitService.java index 9658c4b7..6701e55f 100644 --- a/src/main/java/pro/belbix/ethparser/service/ProfitService.java +++ b/src/main/java/pro/belbix/ethparser/service/ProfitService.java @@ -46,6 +46,10 @@ public Double calculationProfitForPeriod(String address, String start, String en parseLong(start, 0), parseLong(end, Long.MAX_VALUE)); + if (harvestDTOS == null || harvestDTOS.isEmpty()) { + return 0.0; + } + return calculationTotalYield(harvestDTOS.stream() .collect(Collectors.groupingBy(HarvestDTO::getVault, Collectors.toList())), parseLong(start, 0), parseLong(end, Long.MAX_VALUE)); @@ -61,8 +65,7 @@ private Double calculationTotalYield(Map> maps, private Double calcYieldByVault(List v, Long start, Long end) { long startBlockNumber = (v.get(0).getBlock() * start) / v.get(0).getBlockDate(); -// long endBlockNumber = (v.get(v.size() - 1).getBlock() * end) / v.get(v.size() - 1).getBlockDate();// random block number (wip) - Long endBlockNumber = ethBlockService.getLastBlock(v.get(0).getNetwork()); + long endBlockNumber = getEndBlockNumber(v, end); String vaultAddress = v.get(0).getVaultAddress(); String network = v.get(0).getNetwork(); @@ -136,8 +139,20 @@ public Double calculationProfitByVaultForPeriod(String address, String network, parseLong(end, Long.MAX_VALUE), network); + if (harvestDTOS == null || harvestDTOS.isEmpty()) { + return 0.0; + } + return calculationTotalYield(harvestDTOS.stream() .collect(Collectors.groupingBy(HarvestDTO::getVault, Collectors.toList())), parseLong(start, 0), parseLong(end, Long.MAX_VALUE)); } + + private long getEndBlockNumber(List v, Long end) { + if (end == Long.MAX_VALUE) { + return ethBlockService.getLastBlock(v.get(0).getNetwork()); + } + + return (v.get(v.size() - 1).getBlock() * end) / v.get(v.size() - 1).getBlockDate(); + } } From bfb994c57a398e8968e1e012e02347c30b4110d5 Mon Sep 17 00:00:00 2001 From: alex Date: Fri, 7 Jan 2022 23:03:52 +0300 Subject: [PATCH 07/65] fix output param for getReserves in Kyber contracts --- .../ethparser/web3/abi/FunctionsUtils.java | 34 ++++++++++++------- .../web3/abi/FunctionsUtilsTest.java | 29 ++++++++++++++++ src/test/resources/sql/1_eth_contracts.sql | 1 + src/test/resources/sql/5_eth_uni_pairs.sql | 1 + 4 files changed, 53 insertions(+), 12 deletions(-) create mode 100644 src/test/java/pro/belbix/ethparser/web3/abi/FunctionsUtilsTest.java diff --git a/src/main/java/pro/belbix/ethparser/web3/abi/FunctionsUtils.java b/src/main/java/pro/belbix/ethparser/web3/abi/FunctionsUtils.java index 41164068..9ccdd396 100644 --- a/src/main/java/pro/belbix/ethparser/web3/abi/FunctionsUtils.java +++ b/src/main/java/pro/belbix/ethparser/web3/abi/FunctionsUtils.java @@ -25,6 +25,7 @@ import java.util.Optional; import java.util.stream.Collectors; import lombok.extern.log4j.Log4j2; +import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Service; import org.web3j.abi.TypeReference; import org.web3j.abi.datatypes.Address; @@ -60,6 +61,8 @@ public class FunctionsUtils { public final static String TYPE_BOOL = "bool"; private final Map functionsCache = new HashMap<>(); + private final static String EXCLUDE_ONEINCH = "1inch"; + private final static String EXCLUDE_KYBER = "Kyber"; private final Web3Functions web3Functions; private final ContractDbService contractDbService; @@ -76,12 +79,12 @@ public Tuple2 callReserves( Long block, String network) { - String lpName = callStrByName(NAME, lpAddress, block, network).orElse(""); + String lpName = callStrByName(NAME, lpAddress, block, network).orElse(StringUtils.EMPTY); - if (lpName.startsWith("1inch")) { + if (lpName.startsWith(EXCLUDE_ONEINCH)) { return callOneInchReserves(lpAddress, block, network); } else { - return callUniReserves(lpAddress, block, network); + return callUniReserves(lpAddress, block, network, getTypeReferenceForGetReserves(lpName)); } } @@ -110,18 +113,12 @@ private Tuple2 callOneInchReserves(String lpAddress, Long block, return new Tuple2<>(coin0Balance, coin1Balance); } - private Tuple2 callUniReserves(String lpAddress, Long block, String network) { + private Tuple2 callUniReserves(String lpAddress, Long block, String network, List> typeReferences) { List types = web3Functions.callFunction(new Function( GET_RESERVES, Collections.emptyList(), - Arrays.asList(new TypeReference() { - }, - new TypeReference() { - }, - new TypeReference() { - } - )), lpAddress, resolveBlock(block), network); - if (types == null || types.size() < 3) { + typeReferences), lpAddress, resolveBlock(block), network); + if (types == null || types.size() < typeReferences.size()) { log.error("Wrong values for " + lpAddress); return null; } @@ -420,4 +417,17 @@ private static DefaultBlockParameter resolveBlock(Long block) { } return LATEST; } + + private List> getTypeReferenceForGetReserves(String lpName) { + List> typeReferences = new ArrayList<>(Arrays.asList( + new TypeReference() {}, + new TypeReference() {} + )); + + if (!lpName.startsWith(EXCLUDE_KYBER)) { + typeReferences.add(new TypeReference() {}); + } + + return typeReferences; + } } diff --git a/src/test/java/pro/belbix/ethparser/web3/abi/FunctionsUtilsTest.java b/src/test/java/pro/belbix/ethparser/web3/abi/FunctionsUtilsTest.java new file mode 100644 index 00000000..b12e48c2 --- /dev/null +++ b/src/test/java/pro/belbix/ethparser/web3/abi/FunctionsUtilsTest.java @@ -0,0 +1,29 @@ +package pro.belbix.ethparser.web3.abi; + +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; + +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.ContextConfiguration; +import pro.belbix.ethparser.Application; + +@SpringBootTest(classes = Application.class) +@ContextConfiguration +public class FunctionsUtilsTest { + + @Autowired + FunctionsUtils functionsUtils; + + @Test + void callReserves_cyber_with_two_outputParam() { + var result = functionsUtils.callReserves("0xa0fb4487c0935f01cbf9f0274fe3cdb21a965340", null, "matic"); + assertThat(result).isNotNull(); + } + + @Test + void callReserves_uniSwap_with_three_outputParam() { + var result = functionsUtils.callReserves("0x853Ee4b2A13f8a742d64C8F088bE7bA2131f670d", null, "matic"); + assertThat(result).isNotNull(); + } + } diff --git a/src/test/resources/sql/1_eth_contracts.sql b/src/test/resources/sql/1_eth_contracts.sql index 3f90c516..57e49c50 100644 --- a/src/test/resources/sql/1_eth_contracts.sql +++ b/src/test/resources/sql/1_eth_contracts.sql @@ -1148,3 +1148,4 @@ INSERT INTO public.eth_contracts (id, address, created, name, network, type, und INSERT INTO public.eth_contracts (id, address, created, name, network, type, underlying, updated, created_date, updated_date) VALUES (10444, '0x08043b010ad2cd72db75878201fde4b03e28e069', 21718489, 'S_USDC_#V2', 'matic', 5, null, null, 1637697297, null); INSERT INTO public.eth_contracts (id, address, created, name, network, type, underlying, updated, created_date, updated_date) VALUES (10446, '0x9048a123ae6c86a2b9fdc3ea28ed1911de8363a6', 21718513, 'V_WETH', 'matic', 0, null, 22598271, 1637697345, 1639672774); INSERT INTO public.eth_contracts (id, address, created, name, network, type, underlying, updated, created_date, updated_date) VALUES (10447, '0x7ad709df8b897b3b623c5413e92a6f70ca563e1d', 21718518, 'S_WETH', 'matic', 5, null, null, 1637697355, null); +INSERT INTO public.eth_contracts (id, address, created, name, network, type, underlying, updated, created_date, updated_date) VALUES (10448, '0xa0fb4487c0935f01cbf9f0274fe3cdb21a965340', 21718518, 'KyberDMM LP USDC-AUR-0112', 'matic', 2, null, 16645038, 1625597732, 1625761021); diff --git a/src/test/resources/sql/5_eth_uni_pairs.sql b/src/test/resources/sql/5_eth_uni_pairs.sql index 74f28eea..f6b81fb2 100644 --- a/src/test/resources/sql/5_eth_uni_pairs.sql +++ b/src/test/resources/sql/5_eth_uni_pairs.sql @@ -152,3 +152,4 @@ INSERT INTO public.eth_uni_pairs (id, decimals, type, updated_block, contract, t INSERT INTO public.eth_uni_pairs (id, decimals, type, updated_block, contract, token0_id, token1_id) VALUES (10010, 18, 0, 16645038, 10069, 10068, 10063); INSERT INTO public.eth_uni_pairs (id, decimals, type, updated_block, contract, token0_id, token1_id) VALUES (10044, 18, 0, 16657148, 10104, 10063, 10060); INSERT INTO public.eth_uni_pairs (id, decimals, type, updated_block, contract, token0_id, token1_id) VALUES (10008, 18, 0, 16657148, 10062, 10063, 10060); +INSERT INTO public.eth_uni_pairs (id, decimals, type, updated_block, contract, token0_id, token1_id) VALUES (10045, 18, 0, 16657148, 10448, 10063, 10060); From e26a2fcea9111706a656e79f931e28e2496e3526 Mon Sep 17 00:00:00 2001 From: alex Date: Mon, 10 Jan 2022 10:31:39 +0300 Subject: [PATCH 08/65] fix problem with PriceRepository.deleteAllBefore --- .../repositories/v0/PriceRepository.java | 4 ++++ .../service/PriceCleanerServiceTest.java | 22 +++++++++++++++++++ 2 files changed, 26 insertions(+) create mode 100644 src/test/java/pro/belbix/ethparser/service/PriceCleanerServiceTest.java diff --git a/src/main/java/pro/belbix/ethparser/repositories/v0/PriceRepository.java b/src/main/java/pro/belbix/ethparser/repositories/v0/PriceRepository.java index e4402b6c..5427ea21 100644 --- a/src/main/java/pro/belbix/ethparser/repositories/v0/PriceRepository.java +++ b/src/main/java/pro/belbix/ethparser/repositories/v0/PriceRepository.java @@ -3,8 +3,10 @@ import java.util.List; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.springframework.transaction.annotation.Transactional; import pro.belbix.ethparser.dto.v0.PriceDTO; public interface PriceRepository extends JpaRepository { @@ -47,6 +49,8 @@ List fetchLastPrices( + "group by source_address") List fetchAllSourceAddresses(@Param("network") String network); + @Modifying + @Transactional @Query(nativeQuery = true, value = "" + "delete from prices " + "where block_date < :delete_all_before") diff --git a/src/test/java/pro/belbix/ethparser/service/PriceCleanerServiceTest.java b/src/test/java/pro/belbix/ethparser/service/PriceCleanerServiceTest.java new file mode 100644 index 00000000..92e854ed --- /dev/null +++ b/src/test/java/pro/belbix/ethparser/service/PriceCleanerServiceTest.java @@ -0,0 +1,22 @@ +package pro.belbix.ethparser.service; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; + +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.ContextConfiguration; +import pro.belbix.ethparser.Application; + +@SpringBootTest(classes = Application.class) +@ContextConfiguration +public class PriceCleanerServiceTest { + + @Autowired + private PriceCleanerService priceCleanerService; + + @Test + void startPriceCleaner_doesNotThrowJpaSystemException() { + assertDoesNotThrow(() -> priceCleanerService.startPriceCleaner()); + } +} From e2938bed6bee48b08d51839cb6ebe9e1f2f60407 Mon Sep 17 00:00:00 2001 From: alex Date: Wed, 12 Jan 2022 14:00:25 +0300 Subject: [PATCH 09/65] fix problem with UniswapParser and update some logs --- .../java/pro/belbix/ethparser/web3/prices/PriceProvider.java | 2 +- .../ethparser/web3/uniswap/parser/UniswapLpLogParser.java | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/pro/belbix/ethparser/web3/prices/PriceProvider.java b/src/main/java/pro/belbix/ethparser/web3/prices/PriceProvider.java index c5f9b3d0..a24356f4 100644 --- a/src/main/java/pro/belbix/ethparser/web3/prices/PriceProvider.java +++ b/src/main/java/pro/belbix/ethparser/web3/prices/PriceProvider.java @@ -203,7 +203,7 @@ private double getPriceForCoinFromEthLegacy( } } catch (Exception ignore) { } - log.error("Not found lp for {}", address); + log.error("Not found lp for {}, block: {}, network: {}", address, block, network); return 0; } diff --git a/src/main/java/pro/belbix/ethparser/web3/uniswap/parser/UniswapLpLogParser.java b/src/main/java/pro/belbix/ethparser/web3/uniswap/parser/UniswapLpLogParser.java index 3ff3ade8..abedeb0d 100644 --- a/src/main/java/pro/belbix/ethparser/web3/uniswap/parser/UniswapLpLogParser.java +++ b/src/main/java/pro/belbix/ethparser/web3/uniswap/parser/UniswapLpLogParser.java @@ -117,6 +117,10 @@ public UniswapDTO parse(Log ethLog, String network) { //enrich owner TransactionReceipt receipt = web3Functions.fetchTransactionReceipt(dto.getHash(), ETH_NETWORK); + if (receipt == null) { + log.error("TransactionReceipt is null for hash: {}", dto.getHash()); + throw new IllegalStateException("TransactionReceipt is null"); + } dto.setOwner(receipt.getFrom()); //enrich date From e850da070d4fd5d59473a6a2e35b2c6d86c9388d Mon Sep 17 00:00:00 2001 From: alex Date: Thu, 13 Jan 2022 18:56:21 +0300 Subject: [PATCH 10/65] added check status for tx receipt --- .../ethparser/web3/uniswap/parser/UniswapLpLogParser.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/main/java/pro/belbix/ethparser/web3/uniswap/parser/UniswapLpLogParser.java b/src/main/java/pro/belbix/ethparser/web3/uniswap/parser/UniswapLpLogParser.java index abedeb0d..6e7f451e 100644 --- a/src/main/java/pro/belbix/ethparser/web3/uniswap/parser/UniswapLpLogParser.java +++ b/src/main/java/pro/belbix/ethparser/web3/uniswap/parser/UniswapLpLogParser.java @@ -33,6 +33,8 @@ @Log4j2 public class UniswapLpLogParser extends Web3Parser { + private final static String SUCCESS_TX_RESULT = "0x1"; + private final UniswapLpLogDecoder uniswapLpLogDecoder = new UniswapLpLogDecoder(); private final Web3Functions web3Functions; private final Web3Subscriber web3Subscriber; @@ -121,6 +123,10 @@ public UniswapDTO parse(Log ethLog, String network) { log.error("TransactionReceipt is null for hash: {}", dto.getHash()); throw new IllegalStateException("TransactionReceipt is null"); } + + if (!receipt.getStatus().equals(SUCCESS_TX_RESULT)) { + log.warn("Tx {} status is not success", dto.getHash()); + } dto.setOwner(receipt.getFrom()); //enrich date From 4b3f8adf6fbe86a63bda45970d2c33cbc4df30fa Mon Sep 17 00:00:00 2001 From: alex Date: Mon, 31 Jan 2022 11:24:58 +0300 Subject: [PATCH 11/65] added new endpoint for fetching unique owner addresses by network --- .../ethparser/controllers/HarvestController.java | 13 +++++++++++++ .../repositories/v0/HarvestRepository.java | 5 +++++ 2 files changed, 18 insertions(+) diff --git a/src/main/java/pro/belbix/ethparser/controllers/HarvestController.java b/src/main/java/pro/belbix/ethparser/controllers/HarvestController.java index 2fb923a4..651c535f 100644 --- a/src/main/java/pro/belbix/ethparser/controllers/HarvestController.java +++ b/src/main/java/pro/belbix/ethparser/controllers/HarvestController.java @@ -4,6 +4,7 @@ import static pro.belbix.ethparser.utils.CommonUtils.parseLong; import static pro.belbix.ethparser.utils.CommonUtils.reduceListElements; +import com.fasterxml.jackson.databind.ObjectMapper; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.media.ArraySchema; @@ -46,6 +47,7 @@ public class HarvestController { private final VaultActionsDBService vaultActionsDBService; private final ContractDbService contractDbService; private final DtoCache dtoCache; + private final ObjectMapper objectMapper = new ObjectMapper(); public HarvestController(HarvestRepository harvestRepository, VaultActionsDBService vaultActionsDBService, @@ -286,4 +288,15 @@ public RestResponse vaultPeriodOfWork( return RestResponse.ok(periods.get(0).toString()); } + @Operation(summary = "Returns unique owner addresses by network", description = "") + @GetMapping("api/transactions/history/harvest/addresses") + public RestResponse fetchUniqueAddressByNetwork(@RequestParam String network) { + try { + var result = harvestRepository.fetchUniqueAddressByNetwork(network); + return RestResponse.ok(objectMapper.writeValueAsString(result)); + } catch (Exception e) { + return RestResponse.error("Server error"); + } + } + } diff --git a/src/main/java/pro/belbix/ethparser/repositories/v0/HarvestRepository.java b/src/main/java/pro/belbix/ethparser/repositories/v0/HarvestRepository.java index 45d3581b..fd868ae6 100644 --- a/src/main/java/pro/belbix/ethparser/repositories/v0/HarvestRepository.java +++ b/src/main/java/pro/belbix/ethparser/repositories/v0/HarvestRepository.java @@ -307,6 +307,11 @@ Page fetchPagesByVault( + "order by balance desc") List fetchOwnerBalances(@Param("network") String network); + @Query("select h.owner from HarvestDTO h " + + "where h.network = :network " + + "group by h.owner") + List fetchUniqueAddressByNetwork(@Param("network") String network); + interface UserBalance { String getOwner(); From 0501d21f8b6eec536bc82bade31e089442407390 Mon Sep 17 00:00:00 2001 From: alex Date: Sun, 6 Feb 2022 11:47:56 +0300 Subject: [PATCH 12/65] added new tasks for fetching vaults and pools from harvest api --- scripts/sql/schema.sql | 1107 +++++++++++++++++ .../java/pro/belbix/ethparser/AppConfig.java | 12 +- .../model/CovalenthqTransaction.java | 22 + .../ethparser/model/HarvestPoolInfo.java | 25 + .../ethparser/model/HarvestVaultInfo.java | 24 + .../properties/ExternalProperties.java | 27 + .../repositories/eth/ContractRepository.java | 5 + .../service/HarvestPoolInfoService.java | 111 ++ .../service/HarvestVaultInfoService.java | 113 ++ .../service/external/CovalenthqService.java | 58 + .../service/external/HarvestService.java | 33 + .../pro/belbix/ethparser/utils/UrlUtils.java | 20 + 12 files changed, 1555 insertions(+), 2 deletions(-) create mode 100644 scripts/sql/schema.sql create mode 100644 src/main/java/pro/belbix/ethparser/model/CovalenthqTransaction.java create mode 100644 src/main/java/pro/belbix/ethparser/model/HarvestPoolInfo.java create mode 100644 src/main/java/pro/belbix/ethparser/model/HarvestVaultInfo.java create mode 100644 src/main/java/pro/belbix/ethparser/properties/ExternalProperties.java create mode 100644 src/main/java/pro/belbix/ethparser/service/HarvestPoolInfoService.java create mode 100644 src/main/java/pro/belbix/ethparser/service/HarvestVaultInfoService.java create mode 100644 src/main/java/pro/belbix/ethparser/service/external/CovalenthqService.java create mode 100644 src/main/java/pro/belbix/ethparser/service/external/HarvestService.java create mode 100644 src/main/java/pro/belbix/ethparser/utils/UrlUtils.java diff --git a/scripts/sql/schema.sql b/scripts/sql/schema.sql new file mode 100644 index 00000000..2f818536 --- /dev/null +++ b/scripts/sql/schema.sql @@ -0,0 +1,1107 @@ +create user grafana + valid until 'infinity'; +create user backup; +create user hv_dev; + +create table a_eth_address +( + address varchar(255) not null + primary key, + idx bigint + constraint uk_lyyq39oejm6m49nbd9ijhupfg + unique +); +alter table a_eth_address + owner to hv_dev; +grant select on a_eth_address to grafana; +grant select on a_eth_address to backup; + +create table a_eth_hash +( + hash varchar(255) not null + primary key, + idx bigint + constraint uk_eccojqwcjpvg9g6clh6drv36f + unique +); +alter table a_eth_hash + owner to hv_dev; +grant select on a_eth_hash to grafana; +grant select on a_eth_hash to backup; + + +create table a_eth_block +( + number bigint not null + primary key, + author varchar(255), + difficulty varchar(255), + extra_data text, + gas_limit bigint not null, + gas_used bigint not null, + nonce varchar(255), + size bigint not null, + timestamp bigint not null, + total_difficulty varchar(255), + hash bigint + constraint uk_n7r41qxlm5x3rc5vwrpqedhxo + unique + constraint fkffdolbifluijxubt7ff53lf4o + references a_eth_hash (idx), + miner bigint + constraint fk2hme9la57iqv3mj3rgma5d514 + references a_eth_address (idx), + parent_hash bigint + constraint fk5rymjl6kop16ll230q8frrxj9 + references a_eth_hash (idx), + network integer not null +); +alter table a_eth_block + owner to hv_dev; +create index idx_eth_block_hash + on a_eth_block (hash); +create index idx_eth_block_network + on a_eth_block (network); +create index idx_eth_block_timestamp + on a_eth_block (timestamp); +grant select on a_eth_block to grafana; +grant select on a_eth_block to backup; + +create sequence a_eth_tx_id_seq; +create table a_eth_tx +( + id bigserial + primary key, + creates varchar(255), + cumulative_gas_used bigint not null, + gas bigint not null, + gas_price bigint not null, + gas_used bigint not null, + input text, + nonce varchar(255), + public_key varchar(255), + raw varchar(255), + revert_reason varchar(255), + root varchar(255), + status varchar(255), + transaction_index bigint not null, + value varchar(255), + block_number bigint + constraint fkt6utu5q2n3l2cprwxolf4ievh + references a_eth_block + on delete cascade, + contract_address bigint + constraint fkcvpylqyqq9csklhntdsxc3uwf + references a_eth_address (idx), + from_address bigint + constraint fkbwe6xntwov7u176f4ycl6a6gr + references a_eth_address (idx), + hash bigint + constraint idx_eth_txs_hash + unique + constraint fkggohnme0pqf1rbo7ho4bnjj55 + references a_eth_hash (idx), + to_address bigint + constraint fkf7i3tsjk7j6fwjnqq3hh4k8xt + references a_eth_address (idx) +); +alter table a_eth_tx + owner to hv_dev; +grant select on sequence a_eth_tx_id_seq to grafana; +grant select on sequence a_eth_tx_id_seq to backup; +create index idx_eth_txs_block_number + on a_eth_tx (block_number); +create index idx_eth_txs_contract_address + on a_eth_tx (contract_address); +create index idx_eth_txs_from_address + on a_eth_tx (from_address); +create index idx_eth_txs_to_address + on a_eth_tx (to_address); +grant select on a_eth_tx to grafana; +grant select on a_eth_tx to backup; +alter sequence a_eth_tx_id_seq owner to hv_dev; +alter sequence a_eth_tx_id_seq owned by a_eth_tx.id; +grant select on sequence a_eth_tx_id_seq to grafana; +grant select on sequence a_eth_tx_id_seq to backup; + + +create sequence a_eth_log_id_seq; +create table a_eth_log +( + id bigserial + primary key, + data text, + log_id bigint not null, + removed integer not null, + topics text, + transaction_index bigint not null, + type varchar(255), + address bigint + constraint fk7f1b2cyhljaje7ndtadxtdthc + references a_eth_address (idx), + block_number bigint + constraint fkm1bby09fkeo27p1boj9wc1w0r + references a_eth_block, + first_topic bigint + constraint fk515nptp7wer9rq11pi317v0vl + references a_eth_hash (idx), + tx_id bigint + constraint fkdbi2jp2fga70304mo4cjyulc + references a_eth_tx + on delete cascade, + constraint idx_eth_log_tx_id_log_id + unique (tx_id, log_id) +); +alter table a_eth_log + owner to hv_dev; +grant select on sequence a_eth_log_id_seq to grafana; +grant select on sequence a_eth_log_id_seq to backup; +create index idx_eth_log_address + on a_eth_log (address); +create index idx_eth_log_block_number + on a_eth_log (block_number); +create index idx_eth_log_first_topic + on a_eth_log (first_topic); +grant select on a_eth_log to grafana; +grant select on a_eth_log to backup; +alter sequence a_eth_log_id_seq owner to hv_dev; +alter sequence a_eth_log_id_seq owned by a_eth_log.id; +grant select on sequence a_eth_log_id_seq to grafana; +grant select on sequence a_eth_log_id_seq to backup; + + +create sequence b_contract_events_id_seq; +create table b_contract_events +( + id bigserial + primary key, + block bigint + constraint fkta3rg7xqns9jj26rgpyflbxsq + references a_eth_block + on delete cascade, + contract bigint + constraint fkkyifgyelocpd5tnq5bhgra8wk + references a_eth_address (idx), + constraint b_contract_events_block_contract + unique (block, contract) +); +alter table b_contract_events + owner to hv_dev; +grant select on sequence b_contract_events_id_seq to grafana; +grant select on sequence b_contract_events_id_seq to backup; +grant select on b_contract_events to grafana; +grant select on b_contract_events to backup; +alter sequence b_contract_events_id_seq owner to hv_dev; +alter sequence b_contract_events_id_seq owned by b_contract_events.id; +grant select on sequence b_contract_events_id_seq to grafana; +grant select on sequence b_contract_events_id_seq to backup; + +create table b_func_hashes +( + method_id varchar(255) not null + primary key, + name varchar(255) +); +alter table b_func_hashes + owner to hv_dev; +create index b_func_hashes_name + on b_func_hashes (name); +grant select on b_func_hashes to grafana; +grant select on b_func_hashes to backup; + + +create sequence b_contract_txs_id_seq; +create table b_contract_txs +( + id bigserial + primary key, + func_data jsonb, + func_hash varchar(255) + constraint fk15n0nwgc592lqjpue85o8fqqr + references b_func_hashes, + tx_id bigint + constraint b_contract_txs_tx + unique + constraint fk5gv98kcrok11boi2yenwfjkot + references a_eth_tx +); +alter table b_contract_txs + owner to hv_dev; + +grant select on sequence b_contract_txs_id_seq to grafana; +grant select on sequence b_contract_txs_id_seq to backup; +create index b_contract_txs_func_hash + on b_contract_txs (func_hash); +grant select on b_contract_txs to grafana; +grant select on b_contract_txs to backup; +alter sequence b_contract_txs_id_seq owner to hv_dev; +alter sequence b_contract_txs_id_seq owned by b_contract_txs.id; +grant select on sequence b_contract_txs_id_seq to grafana; +grant select on sequence b_contract_txs_id_seq to backup; + + + + +create table b_contract_event_to_tx +( + event_id bigint not null + constraint fkm241gqpq79nknje280fvu30w5 + references b_contract_events, + tx_id bigint not null + constraint fkovb71c3arwj9r2ejp7mmdwblp + references b_contract_txs, + primary key (event_id, tx_id) +); +alter table b_contract_event_to_tx + owner to hv_dev; +grant select on b_contract_event_to_tx to grafana; +grant select on b_contract_event_to_tx to backup; + +create table b_log_hashes +( + method_id varchar(255) not null + primary key, + method_name varchar(255), + topic_hash bigint + constraint fkp9f5yq694sbmt8r1mpb4ha4n0 + references a_eth_hash (idx) +); +alter table b_log_hashes + owner to hv_dev; +create index b_log_hashes_method_name + on b_log_hashes (method_name); +create index b_log_hashes_topic_hash + on b_log_hashes (topic_hash); +grant select on b_log_hashes to grafana; +grant select on b_log_hashes to backup; + +create sequence b_contract_logs_id_seq; +create table b_contract_logs +( + id bigserial + primary key, + log_idx bigint not null, + logs jsonb, + address bigint + constraint fkrkkdnf7mom8emboqbyqfwrdwf + references a_eth_address (idx), + contract_tx_id bigint + constraint fk2wp23byi8261k79hban35a52h + references b_contract_txs + on delete cascade, + topic varchar(255) + constraint fk6aoeyu9exo4d24rrvwqesi3h2 + references b_log_hashes, + constraint b_contract_logs_contract_tx_id_log_idx + unique (contract_tx_id, log_idx) +); +alter table b_contract_logs + owner to hv_dev; +grant select on sequence b_contract_logs_id_seq to grafana; +grant select on sequence b_contract_logs_id_seq to backup; +create index b_contract_logs_address + on b_contract_logs (address); +create index b_contract_logs_topic + on b_contract_logs (topic); +grant select on b_contract_logs to grafana; +grant select on b_contract_logs to backup; +alter sequence b_contract_logs_id_seq owner to hv_dev; +alter sequence b_contract_logs_id_seq owned by b_contract_logs.id; +grant select on sequence b_contract_logs_id_seq to grafana; +grant select on sequence b_contract_logs_id_seq to backup; + +create sequence b_contract_states_id_seq; +create table b_contract_states +( + id bigserial + primary key, + name varchar(255), + value jsonb, + contract_event_id bigint + constraint fknqso7038g1ri26ie22hna1pkm + references b_contract_events + on delete cascade, + constraint uk80s6o9bviindcpv15lpnnyu4x + unique (contract_event_id, name) +); +alter table b_contract_states + owner to hv_dev; +grant select on sequence b_contract_states_id_seq to grafana; +grant select on sequence b_contract_states_id_seq to backup; +grant select on b_contract_states to grafana; +grant select on b_contract_states to backup; +alter sequence b_contract_states_id_seq owner to hv_dev; +alter sequence b_contract_states_id_seq owned by b_contract_states.id; +grant select on sequence b_contract_states_id_seq to grafana; +grant select on sequence b_contract_states_id_seq to backup; + +create table bancor_tx +( + id varchar(255) not null + primary key, + amount double precision, + amount_bnt double precision, + amount_farm double precision, + block bigint, + block_date bigint, + coin varchar(255), + coin_address varchar(255), + farm_as_source boolean, + hash varchar(255), + last_gas double precision, + last_price double precision, + log_id bigint, + other_amount double precision, + other_coin varchar(255), + other_coin_address varchar(255), + owner varchar(255), + price_bnt double precision, + price_farm double precision, + type varchar(255) +); +alter table bancor_tx + owner to hv_dev; + +create table block_cache +( + block bigint not null + primary key, + block_date bigint not null, + network varchar(255) +); + +alter table block_cache + owner to hv_dev; +create index idx_block_cache + on block_cache (block_date); +create index idx_block_cache_net + on block_cache (network); +grant select on block_cache to grafana; +grant select on block_cache to backup; + +create table deployer_tx +( + id varchar(255) not null + primary key, + block bigint not null, + block_date bigint not null, + confirmed integer not null, + from_address varchar(255), + gas_limit numeric(19, 2), + gas_price numeric(19, 2), + gas_used numeric(19, 2), + idx bigint not null, + method_name varchar(255), + to_address varchar(255), + type varchar(255), + value numeric(19, 2), + name varchar(255), + network varchar(255) +); +alter table deployer_tx + owner to hv_dev; +create index idx_deployer_tx + on deployer_tx (block); +create index idx_deployer_tx_network + on deployer_tx (network); +grant select on deployer_tx to grafana; +grant select on deployer_tx to backup; + +create sequence error_parse_id_seq + as integer; +create table error_parse +( + id serial + primary key, + error_class varchar(255), + json text, + network varchar(255), + status integer +); +alter table error_parse + owner to hv_dev; +grant select on sequence error_parse_id_seq to grafana; +grant select on sequence error_parse_id_seq to backup; +grant select on error_parse to grafana; +grant select on error_parse to backup; +alter sequence error_parse_id_seq owner to hv_dev; +alter sequence error_parse_id_seq owned by error_parse.id; +grant select on sequence error_parse_id_seq to grafana; +grant select on sequence error_parse_id_seq to backup; + +create sequence eth_contract_source_codes_id_seq + as integer; +create table eth_contract_source_codes +( + id serial + primary key, + abi text, + address varchar(255) not null, + compiler_version varchar(255), + constructor_arguments text, + contract_name varchar(255) not null, + created_at timestamp, + evmversion varchar(255), + implementation varchar(255), + library varchar(255), + license_type varchar(255), + network varchar(255) not null, + optimization_used boolean, + proxy boolean, + runs varchar(255), + source_code text, + swarm_source varchar(255), + updated_at timestamp, + constraint eth_contract_source_codes_adr_net_pk + unique (address, network) +); +alter table eth_contract_source_codes + owner to hv_dev; +grant select on sequence eth_contract_source_codes_id_seq to grafana; +grant select on sequence eth_contract_source_codes_id_seq to backup; +create index idx_eth_contract_source_codes_address + on eth_contract_source_codes (address); +create index idx_eth_contract_source_codes_name + on eth_contract_source_codes (contract_name); +grant select on eth_contract_source_codes to grafana; +grant select on eth_contract_source_codes to backup; +alter sequence eth_contract_source_codes_id_seq owner to hv_dev; +alter sequence eth_contract_source_codes_id_seq owned by eth_contract_source_codes.id; +grant select on sequence eth_contract_source_codes_id_seq to grafana; +grant select on sequence eth_contract_source_codes_id_seq to backup; + +create sequence eth_contracts_id_seq + as integer; +create table eth_contracts +( + id serial + primary key, + address varchar(255), + created bigint, + name varchar(255), + network varchar(255), + type integer not null, + underlying varchar(255), + updated bigint, + created_date bigint, + updated_date bigint, + constraint eth_c_adr_net_pk + unique (address, network) +); +alter table eth_contracts + owner to hv_dev; +grant select on sequence eth_contracts_id_seq to grafana; +grant select on sequence eth_contracts_id_seq to backup; +create index idx_eth_contracts_address + on eth_contracts (address); +create index idx_eth_contracts_name + on eth_contracts (name); +grant select on eth_contracts to grafana; +grant select on eth_contracts to backup; +alter sequence eth_contracts_id_seq owner to hv_dev; +alter sequence eth_contracts_id_seq owned by eth_contracts.id; +grant select on sequence eth_contracts_id_seq to grafana; +grant select on sequence eth_contracts_id_seq to backup; + +create sequence eth_pools_id_seq + as integer; +create table eth_pools +( + id serial + primary key, + updated_block bigint, + contract integer + constraint uk_5udgya1dmr3v5qtd91fk8wv48 + unique + constraint fk10ejggrjn5iw4rumqvk6mu5yk + references eth_contracts, + controller integer + constraint fkpnih15629ecxme9rgofq22u1q + references eth_contracts, + governance integer + constraint fkik78udlo8lek4dusd872ftko9 + references eth_contracts, + lp_token integer + constraint fk93eitbhw4da8p2a3caom2hck1 + references eth_contracts, + owner integer + constraint fk4su50ff4lc76ng8keety1n2wq + references eth_contracts, + reward_token integer + constraint fk3ldve4891mi7w0hde9kknwalv + references eth_contracts +); +alter table eth_pools + owner to hv_dev; +grant select on sequence eth_pools_id_seq to grafana; +grant select on sequence eth_pools_id_seq to backup; +create index idx_eth_pools + on eth_pools (contract); +grant select on eth_pools to grafana; +grant select on eth_pools to backup; +alter sequence eth_pools_id_seq owner to hv_dev; +alter sequence eth_pools_id_seq owned by eth_pools.id; +grant select on sequence eth_pools_id_seq to grafana; +grant select on sequence eth_pools_id_seq to backup; + +create sequence eth_strategies_id_seq + as integer; +create table eth_strategies +( + id serial + primary key, + updated_block bigint, + contract integer + constraint uk_mm5ag0ivmy35w6u8mwavvk6kh + unique + constraint fkc0kwtg4tlo1el67d0ulxcdkv1 + references eth_contracts +); +alter table eth_strategies + owner to hv_dev; +grant select on sequence eth_strategies_id_seq to grafana; +grant select on sequence eth_strategies_id_seq to backup; +create index idx_eth_strategies_contracts + on eth_strategies (contract); +grant select on eth_strategies to grafana; +grant select on eth_strategies to backup; +alter sequence eth_strategies_id_seq owner to hv_dev; +alter sequence eth_strategies_id_seq owned by eth_strategies.id; +grant select on sequence eth_strategies_id_seq to grafana; +grant select on sequence eth_strategies_id_seq to backup; + +create sequence eth_uni_pairs_id_seq + as integer; +create table eth_uni_pairs +( + id serial + primary key, + decimals bigint, + type integer not null, + updated_block bigint, + contract integer + constraint uk_chgws7al9e6gy2i81674gsmx6 + unique + constraint fkth7ib1cocfiqfp1d0wx5g53ol + references eth_contracts, + token0_id integer + constraint fkt60ljxkxdle0yvh2mnssy5kkr + references eth_contracts, + token1_id integer + constraint fkcacgxuk04t17glca80y7hi38o + references eth_contracts +); +alter table eth_uni_pairs + owner to hv_dev; +grant select on sequence eth_uni_pairs_id_seq to grafana; +grant select on sequence eth_uni_pairs_id_seq to backup; +create index idx_eth_uni_pairs + on eth_uni_pairs (contract); +grant select on eth_uni_pairs to grafana; +grant select on eth_uni_pairs to backup; +alter sequence eth_uni_pairs_id_seq owner to hv_dev; +alter sequence eth_uni_pairs_id_seq owned by eth_uni_pairs.id; +grant select on sequence eth_uni_pairs_id_seq to grafana; +grant select on sequence eth_uni_pairs_id_seq to backup; + +create sequence eth_tokens_id_seq + as integer; +create table eth_tokens +( + id serial + primary key, + decimals bigint, + name varchar(255), + symbol varchar(255), + updated_block bigint, + contract integer + constraint uk_4mjr2gwj0w6ncvr6f0uq04s3q + unique + constraint fkimgaagi5vuont8s0ew6gxqqai + references eth_contracts +); +alter table eth_tokens + owner to hv_dev; +grant select on sequence eth_tokens_id_seq to grafana; +grant select on sequence eth_tokens_id_seq to backup; +create index idx_eth_tokens + on eth_tokens (contract); +grant select on eth_tokens to grafana; +grant select on eth_tokens to backup; +alter sequence eth_tokens_id_seq owner to hv_dev; +alter sequence eth_tokens_id_seq owned by eth_tokens.id; +grant select on sequence eth_tokens_id_seq to grafana; +grant select on sequence eth_tokens_id_seq to backup; + +create sequence eth_token_to_uni_pair_id_seq + as integer; +create table eth_token_to_uni_pair +( + id serial + primary key, + block_start bigint, + token_id integer not null + constraint fki8id42hiwjjwvtifm3ugjon33 + references eth_tokens, + uni_pair_id integer not null + constraint fkrulm04mmvh87yeprexxh1o5xj + references eth_uni_pairs +); +alter table eth_token_to_uni_pair + owner to hv_dev; +grant select on sequence eth_token_to_uni_pair_id_seq to grafana; +grant select on sequence eth_token_to_uni_pair_id_seq to backup; +grant select on eth_token_to_uni_pair to grafana; +grant select on eth_token_to_uni_pair to backup; +alter sequence eth_token_to_uni_pair_id_seq owner to hv_dev; +alter sequence eth_token_to_uni_pair_id_seq owned by eth_token_to_uni_pair.id; +grant select on sequence eth_token_to_uni_pair_id_seq to grafana; +grant select on sequence eth_token_to_uni_pair_id_seq to backup; + +create sequence eth_vaults_id_seq + as integer; +create table eth_vaults +( + id serial + primary key, + decimals bigint, + name varchar(255), + symbol varchar(255), + underlying_unit bigint, + updated_block bigint, + contract integer + constraint uk_jcwcjqrfly836mv8b6i9p6wp6 + unique + constraint fkbj9vb6wndcb0pr44uqwf1rfhw + references eth_contracts, + controller_id integer + constraint fk9unq8mb4dwbt600hj503vr8nj + references eth_contracts, + governance_id integer + constraint fksvfll9hksjiwqodxy2auwdhxv + references eth_contracts, + strategy_id integer + constraint fkohdsk6q4bk3dk52hvykj3533 + references eth_contracts, + underlying_id integer + constraint fkh4gl719e2x448rqt9grxsdx34 + references eth_contracts +); +alter table eth_vaults + owner to hv_dev; +grant select on sequence eth_vaults_id_seq to grafana; +grant select on sequence eth_vaults_id_seq to backup; +create index idx_eth_vaults + on eth_vaults (contract); +grant select on eth_vaults to grafana; +grant select on eth_vaults to backup; +alter sequence eth_vaults_id_seq owner to hv_dev; +alter sequence eth_vaults_id_seq owned by eth_vaults.id; +grant select on sequence eth_vaults_id_seq to grafana; +grant select on sequence eth_vaults_id_seq to backup; + +create table events_tx +( + id varchar(255) not null + primary key, + block bigint, + block_date bigint, + event varchar(255), + hash varchar(255), + info text, + mint_amount double precision, + new_strategy varchar(255), + old_strategy varchar(255), + vault varchar(255), + network varchar(255), + vault_address varchar(255) +); +alter table events_tx + owner to hv_dev; +create index idx_events_network + on events_tx (network); +create index idx_events_tx + on events_tx (block_date); +grant select on events_tx to grafana; +grant select on events_tx to backup; + +create table hard_work +( + id varchar(255) not null + primary key, + all_profit double precision not null, + apr double precision not null, + block bigint not null, + block_date bigint not null, + calls_quantity integer not null, + eth_price double precision not null, + farm_buyback double precision not null, + farm_buyback_eth double precision not null, + farm_buyback_sum double precision not null, + farm_price double precision not null, + fee double precision not null, + fee_eth double precision not null, + full_reward_usd double precision not null, + full_reward_usd_total double precision not null, + gas_used double precision not null, + idle_time bigint not null, + invested double precision not null, + investment_target double precision not null, + perc double precision not null, + period_of_work bigint, + pool_users integer not null, + ps_apr double precision not null, + ps_period_of_work bigint, + ps_tvl_usd double precision not null, + saved_gas_fees double precision not null, + saved_gas_fees_sum double precision not null, + share_change double precision not null, + tvl double precision, + vault varchar(255), + weekly_all_profit double precision not null, + weekly_average_tvl double precision, + weekly_profit double precision not null, + buy_back_rate double precision, + profit_sharing_rate double precision, + auto_stake integer, + network varchar(255), + vault_address varchar(255) +); +alter table hard_work + owner to hv_dev; +create index idx_hard_work + on hard_work (block_date); +create index idx_hard_work_2 + on hard_work (full_reward_usd); +create index idx_hard_work_network + on hard_work (network); +create index idx_hard_work_vault + on hard_work (vault); +create index idx_hard_work_vault_address + on hard_work (vault_address); +grant select on hard_work to grafana; +grant select on hard_work to backup; + +create table harvest_tvl +( + calculate_hash varchar(255) not null + primary key, + calculate_time bigint, + last_all_owners_count integer not null, + last_owners_count integer not null, + last_price double precision, + last_tvl double precision, + network varchar(255) +); +alter table harvest_tvl + owner to hv_dev; +create index idx_harvest_tvl + on harvest_tvl (calculate_time); +grant select on harvest_tvl to grafana; +grant select on harvest_tvl to backup; + +create table harvest_tx +( + id varchar(255) not null + primary key, + all_owners_count integer, + all_pools_owners_count integer, + amount double precision, + amount_in double precision, + block bigint, + block_date bigint, + confirmed integer not null, + hash varchar(255), + last_all_usd_tvl double precision, + last_gas double precision, + last_tvl double precision, + last_usd_tvl double precision, + lp_stat text, + method_name varchar(255), + migrated boolean not null, + owner varchar(255), + owner_balance double precision, + owner_balance_usd double precision, + owner_count integer, + prices text, + profit double precision, + profit_usd double precision, + share_price double precision, + total_amount double precision, + underlying_price double precision, + usd_amount bigint, + vault varchar(255), + network varchar(255), + vault_address varchar(255), + underlying_address varchar(255) +); +alter table harvest_tx + owner to hv_dev; +create index idx_harvest_block_date + on harvest_tx (block_date); +create index idx_harvest_method_name + on harvest_tx (method_name); +create index idx_harvest_network + on harvest_tx (network); +create index idx_harvest_tx + on harvest_tx (block_date); +create index idx_harvest_tx2 + on harvest_tx (method_name, vault); +create index idx_harvest_vault + on harvest_tx (vault); +create index idx_harvest_vault_address + on harvest_tx (vault_address); +grant select on harvest_tx to grafana; +grant select on harvest_tx to backup; + +create table income +( + id varchar(255) not null + primary key, + amount double precision not null, + amount_sum double precision not null, + amount_sum_usd double precision not null, + amount_usd double precision not null, + perc double precision not null, + ps_tvl double precision not null, + ps_tvl_usd double precision not null, + timestamp bigint not null, + week_perc double precision not null +); +alter table income + owner to hv_dev; +create index idx_income + on income (timestamp); +grant select on income to grafana; +grant select on income to backup; + +create table layer_seq +( + seq bigint not null + primary key +); +alter table layer_seq + owner to hv_dev; +grant select on layer_seq to grafana; +grant select on layer_seq to backup; + +create table log_last +( + network varchar(255) not null + primary key, + block bigint +); +alter table log_last + owner to hv_dev; +grant select on log_last to grafana; +grant select on log_last to backup; + + +create table prices +( + id varchar(255) not null + primary key, + block bigint, + block_date bigint, + buy integer, + lp_token0pooled double precision, + lp_token1pooled double precision, + lp_total_supply double precision, + other_token varchar(255), + other_token_amount double precision, + price double precision, + source varchar(255), + token varchar(255), + token_amount double precision, + network varchar(255), + other_token_address varchar(255), + source_address varchar(255), + token_address varchar(255), + owner varchar(255), + recipient varchar(255) +); +alter table prices + owner to hv_dev; +create index idx_prices + on prices (block); +create index idx_prices_network + on prices (network); +create index idx_prices_source + on prices (source); +create index idx_prices_source_address + on prices (source_address); +create index prices_block_date_index + on prices (block_date); +create index prices_network_index + on prices (network); +create index prices_source_index + on prices (source); +grant select on prices to grafana; +grant select on prices to backup; + +create table rewards +( + id varchar(255) not null + primary key, + apy double precision not null, + block bigint not null, + block_date bigint not null, + farm_balance double precision not null, + period_finish bigint not null, + reward double precision not null, + tvl double precision not null, + vault varchar(255), + weekly_apy double precision not null, + network varchar(255), + is_weekly_reward integer, + vault_address varchar(255), + pool_address varchar(255) +); +alter table rewards + owner to hv_dev; +create index idx_rewards + on rewards (block_date); +create index idx_rewards_network + on rewards (network); +create index idx_rewards_vault_address + on rewards (vault_address); +grant select on rewards to grafana; +grant select on rewards to backup; + + +create table strat_info +( + id varchar(255) not null + primary key, + apr double precision, + apy double precision, + block bigint not null, + block_date bigint, + network varchar(255), + percent_of_invested double precision, + percent_of_pool double precision, + platform varchar(255), + pool_address varchar(255), + pool_balance double precision, + pool_extra_info1 varchar(255), + pool_extra_info2 varchar(255), + pool_extra_info3 varchar(255), + pool_specific_underlying varchar(255), + pool_total_supply double precision, + reward_period bigint, + reward_tokens_raw text, + strategy_address varchar(255), + strategy_balance double precision, + strategy_balance_usd double precision, + strategy_created bigint, + strategy_created_date bigint, + strategy_name varchar(255), + strategy_underlying_address varchar(255), + strategy_underlying_name varchar(255), + strategy_underlying_price double precision, + vault_address varchar(255) +); +alter table strat_info + owner to hv_dev; +create index idx_strat_info + on strat_info (block); +create index idx_strat_info_network + on strat_info (network); +create index idx_strat_info_source_vadr + on strat_info (vault_address); +create index idx_strat_info_stadr + on strat_info (strategy_address); +grant select on strat_info to grafana; +grant select on strat_info to backup; + +create table transaction_last +( + network varchar(255) not null + primary key, + block bigint +); +alter table transaction_last + owner to hv_dev; +grant select on transaction_last to grafana; +grant select on transaction_last to backup; + + +create table transfers +( + id varchar(255) not null + primary key, + balance_owner double precision not null, + balance_recipient double precision not null, + block bigint not null, + block_date bigint not null, + method_name varchar(255), + name varchar(255), + owner varchar(255), + price double precision not null, + profit double precision, + profit_usd double precision, + recipient varchar(255), + type varchar(255), + value double precision not null, + network varchar(255), + token_address varchar(255) +); +alter table transfers + owner to hv_dev; +create index idx_transfers_date + on transfers (block_date); +create index idx_transfers_method_name + on transfers (method_name); +create index idx_transfers_name + on transfers (name); +create index idx_transfers_network + on transfers (network); +create index idx_transfers_owner + on transfers (owner); +create index idx_transfers_type + on transfers (type); +grant select on transfers to grafana; +grant select on transfers to backup; + +create table uni_tx +( + id varchar(255) not null + primary key, + amount double precision not null, + block numeric(19, 2), + block_date bigint, + coin varchar(255), + confirmed boolean not null, + hash varchar(255), + last_gas double precision, + last_price double precision, + lp varchar(255), + method_name varchar(255), + other_amount double precision not null, + other_coin varchar(255), + owner varchar(255), + owner_balance double precision, + owner_balance_usd double precision, + owner_count integer, + ps_income_usd double precision, + ps_week_apy double precision, + type varchar(255), + coin_address varchar(255), + lp_address varchar(255), + other_coin_address varchar(255) +); +alter table uni_tx + owner to hv_dev; +create index idx_uni_block_date + on uni_tx (block_date); +create index idx_uni_coin_address + on uni_tx (coin_address); +create index idx_uni_owner + on uni_tx (owner); +create index idx_uni_owner_balance_usd + on uni_tx (owner_balance_usd); +create index idx_uni_tx + on uni_tx (block_date); +grant select on uni_tx to grafana; +grant select on uni_tx to backup; + diff --git a/src/main/java/pro/belbix/ethparser/AppConfig.java b/src/main/java/pro/belbix/ethparser/AppConfig.java index e268fbb0..693a293b 100644 --- a/src/main/java/pro/belbix/ethparser/AppConfig.java +++ b/src/main/java/pro/belbix/ethparser/AppConfig.java @@ -1,24 +1,27 @@ package pro.belbix.ethparser; import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.messaging.simp.config.MessageBrokerRegistry; import org.springframework.scheduling.annotation.EnableScheduling; +import org.springframework.web.client.RestTemplate; import org.springframework.web.socket.config.annotation.EnableWebSocketMessageBroker; import org.springframework.web.socket.config.annotation.StompEndpointRegistry; import org.springframework.web.socket.config.annotation.WebSocketMessageBrokerConfigurer; import pro.belbix.ethparser.properties.AppProperties; import pro.belbix.ethparser.properties.BscAppProperties; import pro.belbix.ethparser.properties.EthAppProperties; +import pro.belbix.ethparser.properties.ExternalProperties; import pro.belbix.ethparser.properties.MaticAppProperties; -import pro.belbix.ethparser.properties.NetworkPropertiesI; @Configuration @EnableConfigurationProperties({ AppProperties.class, EthAppProperties.class, BscAppProperties.class, - MaticAppProperties.class + MaticAppProperties.class, + ExternalProperties.class }) @EnableScheduling public class AppConfig { @@ -41,4 +44,9 @@ public void registerStompEndpoints(StompEndpointRegistry registry) { } } + @Bean + public RestTemplate restTemplate() { + return new RestTemplate(); + } + } diff --git a/src/main/java/pro/belbix/ethparser/model/CovalenthqTransaction.java b/src/main/java/pro/belbix/ethparser/model/CovalenthqTransaction.java new file mode 100644 index 00000000..0ba3c5b6 --- /dev/null +++ b/src/main/java/pro/belbix/ethparser/model/CovalenthqTransaction.java @@ -0,0 +1,22 @@ +package pro.belbix.ethparser.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.List; +import lombok.Data; + +@Data +public class CovalenthqTransaction { + private CovalenthqTransactionItems data; + + @Data + public static class CovalenthqTransactionItems { + private List items; + + @Data + public static class CovalenthqTransactionItem { + @JsonProperty("block_height") + private long blockHeight; + } + } + +} diff --git a/src/main/java/pro/belbix/ethparser/model/HarvestPoolInfo.java b/src/main/java/pro/belbix/ethparser/model/HarvestPoolInfo.java new file mode 100644 index 00000000..026b25bf --- /dev/null +++ b/src/main/java/pro/belbix/ethparser/model/HarvestPoolInfo.java @@ -0,0 +1,25 @@ +package pro.belbix.ethparser.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.List; +import lombok.Data; +import lombok.ToString; + +@Data +@ToString +public class HarvestPoolInfo { + @JsonProperty("eth") + private List ethereumNetwork; + @JsonProperty("matic") + private List maticNetwork; + @JsonProperty("bsc") + private List bscNetwork; + + @Data + @ToString + public static class HarvestPoolItemInfo { + private String contractAddress; + private String type; + private String id; + } +} diff --git a/src/main/java/pro/belbix/ethparser/model/HarvestVaultInfo.java b/src/main/java/pro/belbix/ethparser/model/HarvestVaultInfo.java new file mode 100644 index 00000000..58227b80 --- /dev/null +++ b/src/main/java/pro/belbix/ethparser/model/HarvestVaultInfo.java @@ -0,0 +1,24 @@ +package pro.belbix.ethparser.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.Map; +import lombok.Data; + +@Data +public class HarvestVaultInfo { + @JsonProperty("eth") + private Map ethereumNetwork; + @JsonProperty("matic") + private Map maticNetwork; + @JsonProperty("bsc") + private Map bscNetwork; + + + @Data + public static class HarvestVaultItemInfo { + private String vaultAddress; + private String id; + private String rewardPool; + private String displayName; + } +} diff --git a/src/main/java/pro/belbix/ethparser/properties/ExternalProperties.java b/src/main/java/pro/belbix/ethparser/properties/ExternalProperties.java new file mode 100644 index 00000000..8cdb431a --- /dev/null +++ b/src/main/java/pro/belbix/ethparser/properties/ExternalProperties.java @@ -0,0 +1,27 @@ +package pro.belbix.ethparser.properties; + +import lombok.AccessLevel; +import lombok.Getter; +import lombok.Setter; +import lombok.experimental.FieldDefaults; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.validation.annotation.Validated; + +@Validated +@ConfigurationProperties(prefix = "external") +@FieldDefaults(level = AccessLevel.PRIVATE) +@Getter +@Setter +public class ExternalProperties { + ExternalApi covalenthq; + ExternalApi harvest; + + @FieldDefaults(level = AccessLevel.PRIVATE) + @Getter + @Setter + public static class ExternalApi { + String url; + String key; + } + +} diff --git a/src/main/java/pro/belbix/ethparser/repositories/eth/ContractRepository.java b/src/main/java/pro/belbix/ethparser/repositories/eth/ContractRepository.java index 45d654ca..5558de8a 100644 --- a/src/main/java/pro/belbix/ethparser/repositories/eth/ContractRepository.java +++ b/src/main/java/pro/belbix/ethparser/repositories/eth/ContractRepository.java @@ -46,4 +46,9 @@ List findPoolsByVaultAddress( @Param("network") String network ); + @Query("select c from ContractEntity c " + + "where c.network = :network " + + "and lower(c.address) in(:addresses)") + List findAllByNetworkAndInAddress(String network, List addresses); + } diff --git a/src/main/java/pro/belbix/ethparser/service/HarvestPoolInfoService.java b/src/main/java/pro/belbix/ethparser/service/HarvestPoolInfoService.java new file mode 100644 index 00000000..6f2c9760 --- /dev/null +++ b/src/main/java/pro/belbix/ethparser/service/HarvestPoolInfoService.java @@ -0,0 +1,111 @@ +package pro.belbix.ethparser.service; + +import static pro.belbix.ethparser.service.AbiProviderService.BSC_NETWORK; +import static pro.belbix.ethparser.service.AbiProviderService.ETH_NETWORK; +import static pro.belbix.ethparser.service.AbiProviderService.MATIC_NETWORK; + +import java.util.Collection; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Collectors; +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; +import lombok.experimental.FieldDefaults; +import lombok.extern.slf4j.Slf4j; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import pro.belbix.ethparser.entity.contracts.ContractEntity; +import pro.belbix.ethparser.entity.contracts.PoolEntity; +import pro.belbix.ethparser.model.HarvestPoolInfo.HarvestPoolItemInfo; +import pro.belbix.ethparser.repositories.eth.ContractRepository; +import pro.belbix.ethparser.repositories.eth.PoolRepository; +import pro.belbix.ethparser.service.external.CovalenthqService; +import pro.belbix.ethparser.service.external.HarvestService; +import pro.belbix.ethparser.web3.EthBlockService; +import pro.belbix.ethparser.web3.abi.FunctionsNames; +import pro.belbix.ethparser.web3.abi.FunctionsUtils; +import pro.belbix.ethparser.web3.contracts.ContractLoader; +import pro.belbix.ethparser.web3.contracts.ContractType; + +@Service +@RequiredArgsConstructor +@Slf4j +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) +public class HarvestPoolInfoService { + + HarvestService harvestService; + ContractLoader contractLoader; + ContractRepository contractRepository; + PoolRepository poolRepository; + EthBlockService ethBlockService; + FunctionsUtils functionsUtils; + CovalenthqService covalenthqService; + + + // everyday + @Scheduled(fixedRate = 1000 * 60 * 60 * 24) + public void start() { + try { + var response = harvestService.getPools(); + + List>> poolFutures = List.of( + CompletableFuture.supplyAsync(() -> doTaskByAddressAndNetwork(response.getEthereumNetwork(), ETH_NETWORK)), + CompletableFuture.supplyAsync(() -> doTaskByAddressAndNetwork(response.getMaticNetwork(), MATIC_NETWORK)), + CompletableFuture.supplyAsync(() -> doTaskByAddressAndNetwork(response.getBscNetwork(), BSC_NETWORK)) + ); + + var pools = CompletableFuture.allOf(poolFutures.toArray(new CompletableFuture[0])) + .thenApply(i -> + poolFutures.stream().map(CompletableFuture::join) + .collect(Collectors.toList()) + ) + .get() + .stream() + .flatMap(Collection::stream) + .collect(Collectors.toList()); + + log.info("List of pools: {}", pools); + poolRepository.saveAll(pools); + + } catch (Exception e) { + log.error("Error during getting info from harvest", e); + } + } + + private List doTaskByAddressAndNetwork(List items, String network) { + log.info("Begin find pool and insert in network: {}", network); + var existPools = contractRepository.findAllByNetworkAndInAddress(network, items.stream().map(i -> i.getContractAddress().toLowerCase()).collect(Collectors.toList())); + + var notSavedPools = items.stream() + .filter(i -> existPools.stream().filter(c -> c.getAddress().equalsIgnoreCase(i.getContractAddress())).findFirst().isEmpty()) + .collect(Collectors.toList()); + log.info("Need insert those {}", notSavedPools); + return notSavedPools.stream() + .map(i -> { + try { + log.info("Try to create pool {}", i); + var pool = new PoolEntity(); + var contract = new ContractEntity(); + var block = covalenthqService.getCreatedBlockByLastTransaction(i.getContractAddress(), network); + var createdBlockDate = ethBlockService.getTimestampSecForBlock(block, network);; + contract.setAddress(i.getContractAddress()); + contract.setCreated(block); + contract.setCreatedDate(createdBlockDate); + contract.setNetwork(network); + contract.setName(functionsUtils.callStrByName(FunctionsNames.NAME, i.getContractAddress(), block, network).orElse("")); + contract.setType(ContractType.POOL.getId()); + contract = contractRepository.save(contract); + pool.setContract(contract); + contractLoader.enrichPool(pool, block, network); + log.info("Pool: {}", pool); + return pool; + } catch (Exception e) { + log.error("Can not create pool, {}", i, e); + return null; + } + }) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } +} diff --git a/src/main/java/pro/belbix/ethparser/service/HarvestVaultInfoService.java b/src/main/java/pro/belbix/ethparser/service/HarvestVaultInfoService.java new file mode 100644 index 00000000..7c31a06c --- /dev/null +++ b/src/main/java/pro/belbix/ethparser/service/HarvestVaultInfoService.java @@ -0,0 +1,113 @@ +package pro.belbix.ethparser.service; + +import static pro.belbix.ethparser.service.AbiProviderService.BSC_NETWORK; +import static pro.belbix.ethparser.service.AbiProviderService.ETH_NETWORK; +import static pro.belbix.ethparser.service.AbiProviderService.MATIC_NETWORK; + +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Collectors; +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; +import lombok.experimental.FieldDefaults; +import lombok.extern.slf4j.Slf4j; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import pro.belbix.ethparser.entity.contracts.ContractEntity; +import pro.belbix.ethparser.entity.contracts.VaultEntity; +import pro.belbix.ethparser.model.HarvestVaultInfo.HarvestVaultItemInfo; +import pro.belbix.ethparser.repositories.eth.ContractRepository; +import pro.belbix.ethparser.repositories.eth.VaultRepository; +import pro.belbix.ethparser.service.external.CovalenthqService; +import pro.belbix.ethparser.service.external.HarvestService; +import pro.belbix.ethparser.web3.EthBlockService; +import pro.belbix.ethparser.web3.abi.FunctionsNames; +import pro.belbix.ethparser.web3.abi.FunctionsUtils; +import pro.belbix.ethparser.web3.contracts.ContractLoader; +import pro.belbix.ethparser.web3.contracts.ContractType; + +@Service +@RequiredArgsConstructor +@Slf4j +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) +public class HarvestVaultInfoService { + + HarvestService harvestService; + ContractLoader contractLoader; + ContractRepository contractRepository; + VaultRepository vaultRepository; + EthBlockService ethBlockService; + FunctionsUtils functionsUtils; + CovalenthqService covalenthqService; + + + // everyday + @Scheduled(fixedRate = 1000 * 60 * 60 * 24) + public void start() { + try { + var response = harvestService.getVaults(); + + List>> vaultFutures = List.of( + CompletableFuture.supplyAsync(() -> doTaskByAddressAndNetwork(response.getEthereumNetwork(), ETH_NETWORK)), + CompletableFuture.supplyAsync(() -> doTaskByAddressAndNetwork(response.getMaticNetwork(), MATIC_NETWORK)), + CompletableFuture.supplyAsync(() -> doTaskByAddressAndNetwork(response.getBscNetwork(), BSC_NETWORK)) + ); + + var vaults = CompletableFuture.allOf(vaultFutures.toArray(new CompletableFuture[0])) + .thenApply(i -> + vaultFutures.stream().map(CompletableFuture::join) + .collect(Collectors.toList()) + ) + .get() + .stream() + .flatMap(Collection::stream) + .collect(Collectors.toList()); + + log.info("List of vaults: {}", vaults); + vaultRepository.saveAll(vaults); + + } catch (Exception e) { + log.error("Error during getting info from harvest", e); + } + } + + private List doTaskByAddressAndNetwork(Map items, String network) { + log.info("Begin find vault and insert in network: {}", network); + var existVaults = contractRepository.findAllByNetworkAndInAddress(network, items.values().stream().map(i -> i.getVaultAddress().toLowerCase()).collect(Collectors.toList())); + + var notSavedVaults = items.values().stream() + .filter(i -> existVaults.stream().filter(c -> c.getAddress().equalsIgnoreCase(i.getVaultAddress())).findFirst().isEmpty()) + .collect(Collectors.toList()); + log.info("Need insert those {}", notSavedVaults); + return notSavedVaults.stream() + .map(i -> { + try { + log.info("Try to create pool {}", i); + var vault = new VaultEntity(); + var contract = new ContractEntity(); + var block = covalenthqService.getCreatedBlockByLastTransaction(i.getVaultAddress(), network); + var createdBlockDate = ethBlockService.getTimestampSecForBlock(block, network);; + contract.setAddress(i.getVaultAddress()); + contract.setCreated(block); + contract.setCreatedDate(createdBlockDate); + contract.setNetwork(network); + contract.setName(functionsUtils.callStrByName(FunctionsNames.NAME, i.getVaultAddress(), block, network).orElse("")); + contract.setType(ContractType.POOL.getId()); + contract = contractRepository.save(contract); + vault.setContract(contract); + contractLoader.enrichVault(vault, block, network); + log.info("Vault: {}", vault); + return vault; + } catch (Exception e) { + log.error("Can not create pool, {}", i, e); + return null; + } + }) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } + +} diff --git a/src/main/java/pro/belbix/ethparser/service/external/CovalenthqService.java b/src/main/java/pro/belbix/ethparser/service/external/CovalenthqService.java new file mode 100644 index 00000000..58a7bdba --- /dev/null +++ b/src/main/java/pro/belbix/ethparser/service/external/CovalenthqService.java @@ -0,0 +1,58 @@ +package pro.belbix.ethparser.service.external; + +import static pro.belbix.ethparser.service.AbiProviderService.BSC_NETWORK; +import static pro.belbix.ethparser.service.AbiProviderService.ETH_NETWORK; +import static pro.belbix.ethparser.service.AbiProviderService.MATIC_NETWORK; + +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; +import lombok.experimental.FieldDefaults; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.springframework.web.client.RestTemplate; +import pro.belbix.ethparser.model.CovalenthqTransaction; +import pro.belbix.ethparser.properties.ExternalProperties; +import pro.belbix.ethparser.utils.UrlUtils.CovalenthqUrl; + +@Service +@RequiredArgsConstructor +@Slf4j +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) +public class CovalenthqService { + ExternalProperties externalProperties; + RestTemplate restTemplate; + + + public long getCreatedBlockByLastTransaction(String address, String network) { + var result = getTransactionByAddress(address, network, true, false, 0, 1); + + if (result == null || result.getData() == null || result.getData().getItems() == null || result.getData().getItems().size() != 1) { + return 0L; + } + + return result.getData().getItems().get(0).getBlockHeight(); + } + + public CovalenthqTransaction getTransactionByAddress(String address, String network, boolean isSortAsc, boolean isFullLogs, int page, int limit) { + var url = String.format(CovalenthqUrl.TRANSACTION, externalProperties.getCovalenthq().getUrl(), convertToNetwork(network), address, isSortAsc, isFullLogs, + externalProperties.getCovalenthq().getKey(), page, limit); + try { + return restTemplate.getForObject(url, CovalenthqTransaction.class); + } catch (Exception e) { + log.error("Error during call {}", url, e); + throw new IllegalStateException(e); + } + } + + private String convertToNetwork(String network) { + switch (network) { + case MATIC_NETWORK: + return "137"; + case BSC_NETWORK: + return "56"; + case ETH_NETWORK: + default: + return "1"; + } + } +} diff --git a/src/main/java/pro/belbix/ethparser/service/external/HarvestService.java b/src/main/java/pro/belbix/ethparser/service/external/HarvestService.java new file mode 100644 index 00000000..46e1dbfa --- /dev/null +++ b/src/main/java/pro/belbix/ethparser/service/external/HarvestService.java @@ -0,0 +1,33 @@ +package pro.belbix.ethparser.service.external; + +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; +import lombok.experimental.FieldDefaults; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.springframework.web.client.RestTemplate; +import pro.belbix.ethparser.model.HarvestPoolInfo; +import pro.belbix.ethparser.model.HarvestVaultInfo; +import pro.belbix.ethparser.properties.ExternalProperties; +import pro.belbix.ethparser.utils.UrlUtils.HarvestUrl; + +@Service +@RequiredArgsConstructor +@Slf4j +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) +public class HarvestService { + ExternalProperties externalProperties; + RestTemplate restTemplate; + + public HarvestPoolInfo getPools() { + var url = String.format(HarvestUrl.POOLS, externalProperties.getHarvest().getUrl(), externalProperties.getHarvest().getKey()); + log.info("Starting get pools from harvest {} ", url); + return restTemplate.getForObject(url, HarvestPoolInfo.class); + } + + public HarvestVaultInfo getVaults() { + var url = String.format(HarvestUrl.VAULTS, externalProperties.getHarvest().getUrl(), externalProperties.getHarvest().getKey()); + log.info("Starting get vaults from harvest {} ", url); + return restTemplate.getForObject(url, HarvestVaultInfo.class); + } +} diff --git a/src/main/java/pro/belbix/ethparser/utils/UrlUtils.java b/src/main/java/pro/belbix/ethparser/utils/UrlUtils.java new file mode 100644 index 00000000..48b1001e --- /dev/null +++ b/src/main/java/pro/belbix/ethparser/utils/UrlUtils.java @@ -0,0 +1,20 @@ +package pro.belbix.ethparser.utils; + +public class UrlUtils { + + public interface CovalenthqUrl { + String TRANSACTION = "%s%s/address/%s/transactions_v2/?" + + "quote-currency=USD" + + "&format=JSON" + + "&block-signed-at-asc=%s" + + "&no-logs=%s" + + "&key=%s" + + "&page-number=%s" + + "&page-size=%s"; + } + + public interface HarvestUrl { + String POOLS = "%spools?key=%s"; + String VAULTS = "%svaults?key=%s"; + } +} From c5d953ff96980c3af8ffba94510c2a35fae18564 Mon Sep 17 00:00:00 2001 From: alex Date: Fri, 11 Feb 2022 00:09:53 +0300 Subject: [PATCH 13/65] fix errors, change logic for filter transactions --- pom.xml | 10 +- .../model/CovalenthqTransaction.java | 13 ++- .../repositories/eth/ContractRepository.java | 13 ++- .../ethparser/service/AbiProviderService.java | 2 +- .../service/HarvestPoolInfoService.java | 26 ++++-- .../service/HarvestVaultInfoService.java | 29 ++++-- .../service/external/CovalenthqService.java | 65 ++++++++++--- .../web3/contracts/ContractLoader.java | 24 ++++- .../external/CovalenthqServiceTest.java | 92 +++++++++++++++++++ .../__files/transaction-empty-response.json | 11 +++ .../__files/transaction-simple-response.json | 52 +++++++++++ .../transaction-with-transfer-response.json | 78 ++++++++++++++++ src/test/resources/application.yml | 5 + 13 files changed, 384 insertions(+), 36 deletions(-) create mode 100644 src/test/java/pro/belbix/ethparser/service/external/CovalenthqServiceTest.java create mode 100644 src/test/resources/__files/transaction-empty-response.json create mode 100644 src/test/resources/__files/transaction-simple-response.json create mode 100644 src/test/resources/__files/transaction-with-transfer-response.json diff --git a/pom.xml b/pom.xml index 6103960a..4304882d 100644 --- a/pom.xml +++ b/pom.xml @@ -19,6 +19,7 @@ 11 11 4.8.4 + 3.0.0 -Duser.language=en -Duser.region=US pro.belbix.ethparser.Application @@ -70,7 +71,14 @@ org.springframework.boot spring-boot-starter-actuator - + + + + org.springframework.cloud + spring-cloud-contract-wiremock + ${spring.wiremock.version} + test + diff --git a/src/main/java/pro/belbix/ethparser/model/CovalenthqTransaction.java b/src/main/java/pro/belbix/ethparser/model/CovalenthqTransaction.java index 0ba3c5b6..7cb58d56 100644 --- a/src/main/java/pro/belbix/ethparser/model/CovalenthqTransaction.java +++ b/src/main/java/pro/belbix/ethparser/model/CovalenthqTransaction.java @@ -16,7 +16,18 @@ public static class CovalenthqTransactionItems { public static class CovalenthqTransactionItem { @JsonProperty("block_height") private long blockHeight; + @JsonProperty("log_events") + private List logs; + + @Data + public static class CovalenthqTransactionItemLog { + private CovalenthqTransactionItemLogDecode decoded; + + @Data + public static class CovalenthqTransactionItemLogDecode { + private String name; + } + } } } - } diff --git a/src/main/java/pro/belbix/ethparser/repositories/eth/ContractRepository.java b/src/main/java/pro/belbix/ethparser/repositories/eth/ContractRepository.java index 5558de8a..61dce004 100644 --- a/src/main/java/pro/belbix/ethparser/repositories/eth/ContractRepository.java +++ b/src/main/java/pro/belbix/ethparser/repositories/eth/ContractRepository.java @@ -16,6 +16,14 @@ ContractEntity findFirstByAddress( @Param("network") String network ); + @Query("select t from ContractEntity t " + + "where lower(t.address) = lower(:address) and t.network = :network") + List findFirstByAddress( + @Param("address") String address, + @Param("network") String network, + Pageable pageable + ); + @Query("select t from ContractEntity t " + "where lower(t.address) = lower(:address) and t.type = :type and t.network = :network") ContractEntity findFirstByAddressAndType( @@ -48,7 +56,8 @@ List findPoolsByVaultAddress( @Query("select c from ContractEntity c " + "where c.network = :network " - + "and lower(c.address) in(:addresses)") - List findAllByNetworkAndInAddress(String network, List addresses); + + "and lower(c.address) in(:addresses) " + + "and c.type = :type") + List findAllByNetworkAndInAddressAndType(String network, List addresses, int type); } diff --git a/src/main/java/pro/belbix/ethparser/service/AbiProviderService.java b/src/main/java/pro/belbix/ethparser/service/AbiProviderService.java index e8f058a0..c25246f6 100644 --- a/src/main/java/pro/belbix/ethparser/service/AbiProviderService.java +++ b/src/main/java/pro/belbix/ethparser/service/AbiProviderService.java @@ -134,7 +134,7 @@ private String getUrl(String network) { return ETHERSCAN_URL; } else if (BSC_NETWORK.equals(network)) { return BSC_URL; - }else if (MATIC_NETWORK.equals(network)) { + } else if (MATIC_NETWORK.equals(network)) { return MATIC_URL; } else { throw new IllegalStateException("Unknown network " + network); diff --git a/src/main/java/pro/belbix/ethparser/service/HarvestPoolInfoService.java b/src/main/java/pro/belbix/ethparser/service/HarvestPoolInfoService.java index 6f2c9760..7051fea0 100644 --- a/src/main/java/pro/belbix/ethparser/service/HarvestPoolInfoService.java +++ b/src/main/java/pro/belbix/ethparser/service/HarvestPoolInfoService.java @@ -33,6 +33,7 @@ @Slf4j @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) public class HarvestPoolInfoService { + private static final Long INCREASE_BLOCK_WEIGHT = 1000L; HarvestService harvestService; ContractLoader contractLoader; @@ -43,8 +44,8 @@ public class HarvestPoolInfoService { CovalenthqService covalenthqService; - // everyday - @Scheduled(fixedRate = 1000 * 60 * 60 * 24) + // each hour + @Scheduled(fixedRate = 1000 * 60 * 60) public void start() { try { var response = harvestService.getPools(); @@ -75,7 +76,8 @@ public void start() { private List doTaskByAddressAndNetwork(List items, String network) { log.info("Begin find pool and insert in network: {}", network); - var existPools = contractRepository.findAllByNetworkAndInAddress(network, items.stream().map(i -> i.getContractAddress().toLowerCase()).collect(Collectors.toList())); + var existPools = contractRepository.findAllByNetworkAndInAddressAndType(network, + items.stream().map(i -> i.getContractAddress().toLowerCase()).collect(Collectors.toList()), ContractType.POOL.getId()); var notSavedPools = items.stream() .filter(i -> existPools.stream().filter(c -> c.getAddress().equalsIgnoreCase(i.getContractAddress())).findFirst().isEmpty()) @@ -88,16 +90,28 @@ private List doTaskByAddressAndNetwork(List ite var pool = new PoolEntity(); var contract = new ContractEntity(); var block = covalenthqService.getCreatedBlockByLastTransaction(i.getContractAddress(), network); - var createdBlockDate = ethBlockService.getTimestampSecForBlock(block, network);; + var createdBlockDate = ethBlockService.getTimestampSecForBlock(block, network); + + // in some cases created block is incorrect + var isCorrectBlock = true; + var name = functionsUtils.callStrByName(FunctionsNames.NAME, i.getContractAddress(), block, network).orElse(""); + if (name.isEmpty()) { + isCorrectBlock = false; + name = functionsUtils.callStrByName(FunctionsNames.NAME, i.getContractAddress(), null, network).orElse(""); + } contract.setAddress(i.getContractAddress()); contract.setCreated(block); contract.setCreatedDate(createdBlockDate); contract.setNetwork(network); - contract.setName(functionsUtils.callStrByName(FunctionsNames.NAME, i.getContractAddress(), block, network).orElse("")); + contract.setName(name); contract.setType(ContractType.POOL.getId()); contract = contractRepository.save(contract); pool.setContract(contract); - contractLoader.enrichPool(pool, block, network); + if (isCorrectBlock) { + contractLoader.enrichPool(pool, block, network); + } else { + contractLoader.enrichPool(pool, block + INCREASE_BLOCK_WEIGHT, network); + } log.info("Pool: {}", pool); return pool; } catch (Exception e) { diff --git a/src/main/java/pro/belbix/ethparser/service/HarvestVaultInfoService.java b/src/main/java/pro/belbix/ethparser/service/HarvestVaultInfoService.java index 7c31a06c..2de39a4f 100644 --- a/src/main/java/pro/belbix/ethparser/service/HarvestVaultInfoService.java +++ b/src/main/java/pro/belbix/ethparser/service/HarvestVaultInfoService.java @@ -34,6 +34,7 @@ @Slf4j @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) public class HarvestVaultInfoService { + private static final Long INCREASE_BLOCK_WEIGHT = 1000L; HarvestService harvestService; ContractLoader contractLoader; @@ -44,8 +45,8 @@ public class HarvestVaultInfoService { CovalenthqService covalenthqService; - // everyday - @Scheduled(fixedRate = 1000 * 60 * 60 * 24) + // each hour + @Scheduled(fixedRate = 1000 * 60 * 60) public void start() { try { var response = harvestService.getVaults(); @@ -76,7 +77,8 @@ public void start() { private List doTaskByAddressAndNetwork(Map items, String network) { log.info("Begin find vault and insert in network: {}", network); - var existVaults = contractRepository.findAllByNetworkAndInAddress(network, items.values().stream().map(i -> i.getVaultAddress().toLowerCase()).collect(Collectors.toList())); + var existVaults = contractRepository.findAllByNetworkAndInAddressAndType(network, + items.values().stream().map(i -> i.getVaultAddress().toLowerCase()).collect(Collectors.toList()), ContractType.VAULT.getId()); var notSavedVaults = items.values().stream() .filter(i -> existVaults.stream().filter(c -> c.getAddress().equalsIgnoreCase(i.getVaultAddress())).findFirst().isEmpty()) @@ -89,18 +91,31 @@ private List doTaskByAddressAndNetwork(Map CHAIN_BY_NETWORK = Map.of( + MATIC_NETWORK, "137", + ETH_NETWORK, "1", + BSC_NETWORK, "56" + ); + ExternalProperties externalProperties; RestTemplate restTemplate; public long getCreatedBlockByLastTransaction(String address, String network) { - var result = getTransactionByAddress(address, network, true, false, 0, 1); + try { + var page = 0; + var result = getTransactionByAddress(address, network, true, false, page, TRANSFER_LIMIT); - if (result == null || result.getData() == null || result.getData().getItems() == null || result.getData().getItems().size() != 1) { - return 0L; - } + if (result == null || result.getData() == null || result.getData().getItems() == null) { + return 0; + } - return result.getData().getItems().get(0).getBlockHeight(); + var createdTx = findCovalenthqTransactionItem(result); + + while (createdTx == null) { + Thread.sleep(100); + page++; + result = getTransactionByAddress(address, network, true, false, page, TRANSFER_LIMIT); + if (result == null || result.getData() == null || result.getData().getItems() == null || result.getData().getItems().isEmpty()) { + return 0; + } + createdTx = findCovalenthqTransactionItem(result); + } + + return createdTx.getBlockHeight(); + } catch (Exception e) { + log.error("Error during call getCreatedBlockByLastTransaction", e); + return 0; + } } public CovalenthqTransaction getTransactionByAddress(String address, String network, boolean isSortAsc, boolean isFullLogs, int page, int limit) { @@ -45,14 +76,22 @@ public CovalenthqTransaction getTransactionByAddress(String address, String netw } private String convertToNetwork(String network) { - switch (network) { - case MATIC_NETWORK: - return "137"; - case BSC_NETWORK: - return "56"; - case ETH_NETWORK: - default: - return "1"; + return Optional.ofNullable(CHAIN_BY_NETWORK.get(network)).orElse("1"); + } + + private CovalenthqTransactionItem findCovalenthqTransactionItem(CovalenthqTransaction result) { + return result.getData().getItems().stream() + .filter(items -> items.getLogs() != null && notHasTransferInLog(items.getLogs())) + .findFirst() + .orElse(null); + } + + private boolean notHasTransferInLog(List logs) { + for (CovalenthqTransactionItemLog log : logs) { + if (log.getDecoded() != null && TRANSFER_LOG_NAME.equals(log.getDecoded().getName())) { + return false; + } } + return true; } } diff --git a/src/main/java/pro/belbix/ethparser/web3/contracts/ContractLoader.java b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractLoader.java index 0284f05d..6046ee20 100644 --- a/src/main/java/pro/belbix/ethparser/web3/contracts/ContractLoader.java +++ b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractLoader.java @@ -26,6 +26,7 @@ import java.util.Map.Entry; import lombok.extern.log4j.Log4j2; import org.apache.logging.log4j.util.Strings; +import org.springframework.data.domain.PageRequest; import org.springframework.stereotype.Service; import pro.belbix.ethparser.entity.contracts.ContractEntity; import pro.belbix.ethparser.entity.contracts.PoolEntity; @@ -211,6 +212,15 @@ private void enrichToken(TokenEntity tokenEntity, long block, String network) { } public void enrichVault(VaultEntity vaultEntity, long block, String network) { + enrichVault(vaultEntity, block, block, network); + } + + public void enrichVaultWithLatestBlock(VaultEntity vaultEntity, long block, String network) { + enrichVault(vaultEntity, block, null, network); + } + + + public void enrichVault(VaultEntity vaultEntity, Long block, Long blockForProperty, String network) { if (appProperties.isOnlyApi()) { return; } @@ -256,14 +266,14 @@ public void enrichVault(VaultEntity vaultEntity, long block, String network) { network )); vaultEntity.setName( - functionsUtils.callStrByName(FunctionsNames.NAME, address, block, network).orElse("")); + functionsUtils.callStrByName(FunctionsNames.NAME, address, blockForProperty, network).orElse("")); vaultEntity.setSymbol( - functionsUtils.callStrByName(FunctionsNames.SYMBOL, address, block, network).orElse("")); + functionsUtils.callStrByName(FunctionsNames.SYMBOL, address, blockForProperty, network).orElse("")); vaultEntity.setDecimals( - functionsUtils.callIntByName(FunctionsNames.DECIMALS, address, block, network) + functionsUtils.callIntByName(FunctionsNames.DECIMALS, address, blockForProperty, network) .orElse(BigInteger.ZERO).longValue()); vaultEntity.setUnderlyingUnit( - functionsUtils.callIntByName(FunctionsNames.UNDERLYING_UNIT, address, block, network) + functionsUtils.callIntByName(FunctionsNames.UNDERLYING_UNIT, address, blockForProperty, network) .orElse(BigInteger.ZERO).longValue()); } @@ -492,7 +502,11 @@ ContractEntity findOrCreateContract(String address, || ZERO_ADDRESS.equalsIgnoreCase(address)) { return null; } - ContractEntity entity = contractRepository.findFirstByAddress(address, network); + var result = contractRepository.findFirstByAddress(address, network, PageRequest.of(0, 1)); + if (result == null || result.isEmpty()) { + return null; + } + ContractEntity entity = result.get(0); if (appProperties.isOnlyApi()) { return entity; } diff --git a/src/test/java/pro/belbix/ethparser/service/external/CovalenthqServiceTest.java b/src/test/java/pro/belbix/ethparser/service/external/CovalenthqServiceTest.java new file mode 100644 index 00000000..05e3aaf5 --- /dev/null +++ b/src/test/java/pro/belbix/ethparser/service/external/CovalenthqServiceTest.java @@ -0,0 +1,92 @@ +package pro.belbix.ethparser.service.external; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.stubFor; +import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; +import static org.apache.http.HttpHeaders.CONTENT_TYPE; +import static org.assertj.core.api.Assertions.assertThat; +import static pro.belbix.ethparser.service.AbiProviderService.ETH_NETWORK; + +import com.github.tomakehurst.wiremock.stubbing.Scenario; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.cloud.contract.wiremock.AutoConfigureWireMock; +import org.springframework.test.context.TestPropertySource; + +@SpringBootTest +@AutoConfigureWireMock(port = 801) +@TestPropertySource(properties = { + "external.covalenthq.url=http://localhost:801/", + "external.covalenthq.key=1" +}) +public class CovalenthqServiceTest { + private static final String TRANSACTION_URL_PART = "/1/address/0x1234/transactions_v2/?quote-currency=USD&format=JSON&block-signed-at-asc=true&no-logs=false&key=1&page-number=0&page-size=3"; + private static final String TRANSACTION_SECOND_PAGE_URL_PART = "/1/address/0x1234/transactions_v2/?quote-currency=USD&format=JSON&block-signed-at-asc=true&no-logs=false&key=1&page-number=1&page-size=3"; + private static final String ADDRESS = "0x1234"; + private static final long BLOCK = 111111; + + @Autowired + private CovalenthqService covalenthqService; + + + @Test + void testNotNull() { + assertThat(covalenthqService).isNotNull(); + } + + @Test + void getCreatedBlockByLastTransaction_successResult() { + stubFor( + get(urlEqualTo(TRANSACTION_URL_PART)) + .willReturn(aResponse() + .withHeader(CONTENT_TYPE, "application/json") + .withBodyFile("transaction-simple-response.json") + ) + ); + var result = covalenthqService.getCreatedBlockByLastTransaction(ADDRESS, ETH_NETWORK); + assertThat(result).isEqualTo(BLOCK); + } + + @Test + void getCreatedBlockByLastTransaction_emptyResult() { + stubFor( + get(urlEqualTo(TRANSACTION_URL_PART)) + .willReturn(aResponse() + .withHeader(CONTENT_TYPE, "application/json") + .withBodyFile("transaction-empty-response.json") + ) + ); + var result = covalenthqService.getCreatedBlockByLastTransaction(ADDRESS, ETH_NETWORK); + assertThat(result).isEqualTo(0); + } + + @Test + void getCreatedBlockByLastTransaction_withTwoCall() { + stubFor( + get(urlEqualTo(TRANSACTION_URL_PART)) + .inScenario("transfer") + .whenScenarioStateIs(Scenario.STARTED) + .willSetStateTo("second") + .willReturn(aResponse() + .withHeader(CONTENT_TYPE, "application/json") + .withBodyFile("transaction-with-transfer-response.json") + ) + ); + stubFor( + get(urlEqualTo(TRANSACTION_SECOND_PAGE_URL_PART)) + .inScenario("transfer") + .whenScenarioStateIs("second") + .willSetStateTo(Scenario.STARTED) + .willReturn(aResponse() + .withHeader(CONTENT_TYPE, "application/json") + .withBodyFile("transaction-simple-response.json") + ) + ); + + var result = covalenthqService.getCreatedBlockByLastTransaction(ADDRESS, ETH_NETWORK); + assertThat(result).isEqualTo(BLOCK); + } + +} diff --git a/src/test/resources/__files/transaction-empty-response.json b/src/test/resources/__files/transaction-empty-response.json new file mode 100644 index 00000000..2a8cbc18 --- /dev/null +++ b/src/test/resources/__files/transaction-empty-response.json @@ -0,0 +1,11 @@ +{ + "data": { + "address": "0x2fee56e039acccefa3cb1f3051ad00fe550a472c", + "updated_at": "2022-02-09T13:31:25.293424937Z", + "next_update_at": "2022-02-09T13:36:25.293425537Z", + "quote_currency": "USD", + "chain_id": 137, + "items": [ + ] + } +} \ No newline at end of file diff --git a/src/test/resources/__files/transaction-simple-response.json b/src/test/resources/__files/transaction-simple-response.json new file mode 100644 index 00000000..ca5f8e1d --- /dev/null +++ b/src/test/resources/__files/transaction-simple-response.json @@ -0,0 +1,52 @@ +{ + "data": { + "address": "0x2fee56e039acccefa3cb1f3051ad00fe550a472c", + "updated_at": "2022-02-09T13:31:25.293424937Z", + "next_update_at": "2022-02-09T13:36:25.293425537Z", + "quote_currency": "USD", + "chain_id": 137, + "items": [ + { + "block_signed_at": "2021-08-16T22:31:56Z", + "block_height": 111111, + "tx_hash": "0x4c60c231cdac354061b9c78b587f67b037636655ffcf1fb527422c2db531d818", + "tx_offset": 173, + "successful": true, + "from_address": "0x36ec37f30459eaa6347a8d5b052c559d131f4c41", + "from_address_label": null, + "to_address": "0x8ae127d224094cb1b27e1b28a472e588cbcc7620", + "to_address_label": null, + "value": "0", + "value_quote": 0.0, + "gas_offered": 51675, + "gas_spent": 51675, + "gas_price": 1000000001, + "gas_quote": 7.534433406675268E-5, + "gas_quote_rate": 1.4580422639846802, + "log_events": [ + { + "block_signed_at": "2021-08-16T22:31:56Z", + "block_height": 111111, + "tx_offset": 173, + "log_offset": 380, + "tx_hash": "0x4c60c231cdac354061b9c78b587f67b037636655ffcf1fb527422c2db531d818", + "raw_log_topics": [ + "0x4dfe1bbbcf077ddc3e01291eea2d5c70c2b422b415d95645b9adcfd678cb1d63", + "0x0000000000000000000000000000000000000000000000000000000000001010", + "0x00000000000000000000000036ec37f30459eaa6347a8d5b052c559d131f4c41", + "0x000000000000000000000000e7e2cb8c81c10ff191a73fe266788c9ce62ec754" + ], + "sender_contract_decimals": 18, + "sender_name": "Matic Token", + "sender_contract_ticker_symbol": "MATIC", + "sender_address": "0x0000000000000000000000000000000000001010", + "sender_address_label": null, + "sender_logo_url": "https://logos.covalenthq.com/tokens/0x7d1afa7b718fb893db30a3abc0cfc608aacfebb0.png", + "raw_log_data": "0x00000000000000000000000000000000000000000000000000002eff860597db00000000000000000000000000000000000000000000000198ecc30b0c3841ea000000000000000000000000000000000000000000000ac6dd8b15f654a4b5c400000000000000000000000000000000000000000000000198ec940b8632aa0f000000000000000000000000000000000000000000000ac6dd8b44f5daaa4d9f", + "decoded": null + } + ] + } + ] + } +} \ No newline at end of file diff --git a/src/test/resources/__files/transaction-with-transfer-response.json b/src/test/resources/__files/transaction-with-transfer-response.json new file mode 100644 index 00000000..8b158d17 --- /dev/null +++ b/src/test/resources/__files/transaction-with-transfer-response.json @@ -0,0 +1,78 @@ +{ + "data": { + "address": "0x2fee56e039acccefa3cb1f3051ad00fe550a472c", + "updated_at": "2022-02-09T13:31:25.293424937Z", + "next_update_at": "2022-02-09T13:36:25.293425537Z", + "quote_currency": "USD", + "chain_id": 137, + "items": [ + { + "block_signed_at": "2021-08-16T22:31:56Z", + "block_height": 111111, + "tx_hash": "0x4c60c231cdac354061b9c78b587f67b037636655ffcf1fb527422c2db531d818", + "tx_offset": 173, + "successful": true, + "from_address": "0x36ec37f30459eaa6347a8d5b052c559d131f4c41", + "from_address_label": null, + "to_address": "0x8ae127d224094cb1b27e1b28a472e588cbcc7620", + "to_address_label": null, + "value": "0", + "value_quote": 0.0, + "gas_offered": 51675, + "gas_spent": 51675, + "gas_price": 1000000001, + "gas_quote": 7.534433406675268E-5, + "gas_quote_rate": 1.4580422639846802, + "log_events": [ + { + "block_signed_at": "2021-08-16T22:31:56Z", + "block_height": 111111, + "tx_offset": 173, + "log_offset": 380, + "tx_hash": "0x4c60c231cdac354061b9c78b587f67b037636655ffcf1fb527422c2db531d818", + "raw_log_topics": [ + "0x4dfe1bbbcf077ddc3e01291eea2d5c70c2b422b415d95645b9adcfd678cb1d63", + "0x0000000000000000000000000000000000000000000000000000000000001010", + "0x00000000000000000000000036ec37f30459eaa6347a8d5b052c559d131f4c41", + "0x000000000000000000000000e7e2cb8c81c10ff191a73fe266788c9ce62ec754" + ], + "sender_contract_decimals": 18, + "sender_name": "Matic Token", + "sender_contract_ticker_symbol": "MATIC", + "sender_address": "0x0000000000000000000000000000000000001010", + "sender_address_label": null, + "sender_logo_url": "https://logos.covalenthq.com/tokens/0x7d1afa7b718fb893db30a3abc0cfc608aacfebb0.png", + "raw_log_data": "0x00000000000000000000000000000000000000000000000000002eff860597db00000000000000000000000000000000000000000000000198ecc30b0c3841ea000000000000000000000000000000000000000000000ac6dd8b15f654a4b5c400000000000000000000000000000000000000000000000198ec940b8632aa0f000000000000000000000000000000000000000000000ac6dd8b44f5daaa4d9f", + "decoded": { + "name": "Transfer", + "signature": "Transfer(indexed address from, indexed address to, uint256 value)", + "params": [ + { + "name": "from", + "type": "address", + "indexed": true, + "decoded": true, + "value": "0x36ec37f30459eaa6347a8d5b052c559d131f4c41" + }, + { + "name": "to", + "type": "address", + "indexed": true, + "decoded": true, + "value": "0x2fee56e039acccefa3cb1f3051ad00fe550a472c" + }, + { + "name": "value", + "type": "uint256", + "indexed": false, + "decoded": true, + "value": "26347543096503948200000000000" + } + ] + } + } + ] + } + ] + } +} \ No newline at end of file diff --git a/src/test/resources/application.yml b/src/test/resources/application.yml index 04d95696..c333d189 100644 --- a/src/test/resources/application.yml +++ b/src/test/resources/application.yml @@ -47,3 +47,8 @@ logging: level: com.yannbriancon.interceptor.HibernateQueryInterceptor: OFF # pro.belbix: debug + +# only for m1 Mac users +wiremock: + server: + httpsPort: -1 \ No newline at end of file From f066a28fd595eb7e8a6506580e019b9f03b6cf92 Mon Sep 17 00:00:00 2001 From: alex Date: Mon, 21 Feb 2022 20:45:58 +0300 Subject: [PATCH 14/65] Fetch transactions from covalenthq api --- scripts/sql/schema.sql | 17 ++ .../profit/CovalenthqVaultTransaction.java | 35 +++++ .../CovalenthqVaultTransactionType.java | 6 + .../model/CovalenthqTransaction.java | 33 ---- ...ovalenthqTransactionByContractAddress.java | 26 ++++ .../model/CovalenthqTransactionHistory.java | 65 ++++++++ .../belbix/ethparser/model/LogEventParam.java | 18 +++ .../CovalenthqVaultTransactionRepository.java | 10 ++ .../repositories/v0/HarvestRepository.java | 2 + .../ethparser/service/ProfitService.java | 14 ++ .../service/external/CovalenthqService.java | 44 ++++-- .../task/CovalenthqTransactionTask.java | 146 ++++++++++++++++++ .../HarvestPoolInfoTask.java} | 4 +- .../HarvestVaultInfoTask.java} | 4 +- .../pro/belbix/ethparser/utils/UrlUtils.java | 35 ++++- 15 files changed, 402 insertions(+), 57 deletions(-) create mode 100644 src/main/java/pro/belbix/ethparser/entity/profit/CovalenthqVaultTransaction.java create mode 100644 src/main/java/pro/belbix/ethparser/entity/profit/CovalenthqVaultTransactionType.java delete mode 100644 src/main/java/pro/belbix/ethparser/model/CovalenthqTransaction.java create mode 100644 src/main/java/pro/belbix/ethparser/model/CovalenthqTransactionByContractAddress.java create mode 100644 src/main/java/pro/belbix/ethparser/model/CovalenthqTransactionHistory.java create mode 100644 src/main/java/pro/belbix/ethparser/model/LogEventParam.java create mode 100644 src/main/java/pro/belbix/ethparser/repositories/covalenthq/CovalenthqVaultTransactionRepository.java create mode 100644 src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java rename src/main/java/pro/belbix/ethparser/service/{HarvestPoolInfoService.java => task/HarvestPoolInfoTask.java} (98%) rename src/main/java/pro/belbix/ethparser/service/{HarvestVaultInfoService.java => task/HarvestVaultInfoTask.java} (98%) diff --git a/scripts/sql/schema.sql b/scripts/sql/schema.sql index 2f818536..9a5da908 100644 --- a/scripts/sql/schema.sql +++ b/scripts/sql/schema.sql @@ -1105,3 +1105,20 @@ create index idx_uni_tx grant select on uni_tx to grafana; grant select on uni_tx to backup; + +create table covalenthq_vault_tx +( + id bigserial + primary key, + network varchar(255), + block numeric(19, 2), + transaction_hash varchar(255), + contract_decimal numeric(19, 2), + contract_address varchar(255), + owner_address varchar(255), + value numeric(19, 2), + signed_at timestamp, + type varchar(255) +); + + diff --git a/src/main/java/pro/belbix/ethparser/entity/profit/CovalenthqVaultTransaction.java b/src/main/java/pro/belbix/ethparser/entity/profit/CovalenthqVaultTransaction.java new file mode 100644 index 00000000..cd74dac5 --- /dev/null +++ b/src/main/java/pro/belbix/ethparser/entity/profit/CovalenthqVaultTransaction.java @@ -0,0 +1,35 @@ +package pro.belbix.ethparser.entity.profit; + + +import java.time.LocalDateTime; +import javax.persistence.Entity; +import javax.persistence.EnumType; +import javax.persistence.Enumerated; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; +import lombok.AccessLevel; +import lombok.Data; +import lombok.experimental.FieldDefaults; + +@Entity +@Table(name = "covalenthq_vault_tx") +@Data +@FieldDefaults(level = AccessLevel.PRIVATE) +public class CovalenthqVaultTransaction { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + Integer id; + String network; + long block; + String transactionHash; + int contractDecimal; + String contractAddress; + String ownerAddress; + long value; + LocalDateTime signedAt; + @Enumerated(EnumType.STRING) + CovalenthqVaultTransactionType type; +} diff --git a/src/main/java/pro/belbix/ethparser/entity/profit/CovalenthqVaultTransactionType.java b/src/main/java/pro/belbix/ethparser/entity/profit/CovalenthqVaultTransactionType.java new file mode 100644 index 00000000..27c941f0 --- /dev/null +++ b/src/main/java/pro/belbix/ethparser/entity/profit/CovalenthqVaultTransactionType.java @@ -0,0 +1,6 @@ +package pro.belbix.ethparser.entity.profit; + +public enum CovalenthqVaultTransactionType { + WITHDRAW, + DEPOSIT +} diff --git a/src/main/java/pro/belbix/ethparser/model/CovalenthqTransaction.java b/src/main/java/pro/belbix/ethparser/model/CovalenthqTransaction.java deleted file mode 100644 index 7cb58d56..00000000 --- a/src/main/java/pro/belbix/ethparser/model/CovalenthqTransaction.java +++ /dev/null @@ -1,33 +0,0 @@ -package pro.belbix.ethparser.model; - -import com.fasterxml.jackson.annotation.JsonProperty; -import java.util.List; -import lombok.Data; - -@Data -public class CovalenthqTransaction { - private CovalenthqTransactionItems data; - - @Data - public static class CovalenthqTransactionItems { - private List items; - - @Data - public static class CovalenthqTransactionItem { - @JsonProperty("block_height") - private long blockHeight; - @JsonProperty("log_events") - private List logs; - - @Data - public static class CovalenthqTransactionItemLog { - private CovalenthqTransactionItemLogDecode decoded; - - @Data - public static class CovalenthqTransactionItemLogDecode { - private String name; - } - } - } - } -} diff --git a/src/main/java/pro/belbix/ethparser/model/CovalenthqTransactionByContractAddress.java b/src/main/java/pro/belbix/ethparser/model/CovalenthqTransactionByContractAddress.java new file mode 100644 index 00000000..901e62d9 --- /dev/null +++ b/src/main/java/pro/belbix/ethparser/model/CovalenthqTransactionByContractAddress.java @@ -0,0 +1,26 @@ +package pro.belbix.ethparser.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + +@Data +public class CovalenthqTransactionByContractAddress { + private CovalenthqTransactionByContractAddressData data; + + @Data + public static class CovalenthqTransactionByContractAddressData { + private CovalenthqTransactionByContractAddressDataItems items; + + @Data + public static class CovalenthqTransactionByContractAddressDataItems { + @JsonProperty("tx_hash") + private String hash; + private boolean successful; + @JsonProperty("from_address") + private String fromAddress; + @JsonProperty("to_address") + private String toAddress; + + } + } +} diff --git a/src/main/java/pro/belbix/ethparser/model/CovalenthqTransactionHistory.java b/src/main/java/pro/belbix/ethparser/model/CovalenthqTransactionHistory.java new file mode 100644 index 00000000..7430614d --- /dev/null +++ b/src/main/java/pro/belbix/ethparser/model/CovalenthqTransactionHistory.java @@ -0,0 +1,65 @@ +package pro.belbix.ethparser.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import java.time.LocalDateTime; +import java.util.List; +import lombok.Data; + +@Data +public class CovalenthqTransactionHistory { + private CovalenthqTransactionHistoryItems data; + + @Data + public static class CovalenthqTransactionHistoryItems { + private List items; + private CovalenthqTransactionHistoryPagination pagination; + + @Data + public static class CovalenthqTransactionHistoryItem { + @JsonProperty("block_signed_at") + private LocalDateTime signedAt; + @JsonProperty("block_height") + private long blockHeight; + @JsonProperty("tx_hash") + private String transactionHash; + @JsonProperty("from_address") + private String fromAddress; + @JsonProperty("to_address") + private String toAddress; + @JsonProperty("log_events") + private List logs; + + @Data + public static class CovalenthqTransactionHistoryItemLog { + @JsonProperty("block_height") + private long blockHeight; + @JsonProperty("tx_hash") + private String transactionHash; + private CovalenthqTransactionHistoryItemLogDecode decoded; + @JsonProperty("sender_contract_decimals") + private int contractDecimal; + + @Data + public static class CovalenthqTransactionHistoryItemLogDecode { + private String name; + private List params; + + @Data + public static class CovalenthqTransactionHistoryItemLogDecodeParam { + private String name; + private String type; + private boolean indexed; + private boolean decoded; + private String value; + } + } + } + } + + @Data + public static class CovalenthqTransactionHistoryPagination { + @JsonProperty("has_more") + private boolean hasMore; + } + } +} diff --git a/src/main/java/pro/belbix/ethparser/model/LogEventParam.java b/src/main/java/pro/belbix/ethparser/model/LogEventParam.java new file mode 100644 index 00000000..32c3cc1f --- /dev/null +++ b/src/main/java/pro/belbix/ethparser/model/LogEventParam.java @@ -0,0 +1,18 @@ +package pro.belbix.ethparser.model; + +import lombok.AccessLevel; +import lombok.experimental.FieldDefaults; + +@FieldDefaults(makeFinal = true, level = AccessLevel.PUBLIC) +public enum LogEventParam { + WITHDRAW_TO("provider"), + WITHDRAW_VALUE("value"), + DEPOSIT_FROM("dst"), + DEPOSIT_VALUE("wad"); + + LogEventParam(String value) { + this.value = value; + } + + String value; +} diff --git a/src/main/java/pro/belbix/ethparser/repositories/covalenthq/CovalenthqVaultTransactionRepository.java b/src/main/java/pro/belbix/ethparser/repositories/covalenthq/CovalenthqVaultTransactionRepository.java new file mode 100644 index 00000000..966dcf2d --- /dev/null +++ b/src/main/java/pro/belbix/ethparser/repositories/covalenthq/CovalenthqVaultTransactionRepository.java @@ -0,0 +1,10 @@ +package pro.belbix.ethparser.repositories.covalenthq; + +import java.util.List; +import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.repository.JpaRepository; +import pro.belbix.ethparser.entity.profit.CovalenthqVaultTransaction; + +public interface CovalenthqVaultTransactionRepository extends JpaRepository { + List findAllByNetworkAndContractAddress(String network, String contractAddress, Pageable pageable); +} diff --git a/src/main/java/pro/belbix/ethparser/repositories/v0/HarvestRepository.java b/src/main/java/pro/belbix/ethparser/repositories/v0/HarvestRepository.java index fd868ae6..a4f1d9c0 100644 --- a/src/main/java/pro/belbix/ethparser/repositories/v0/HarvestRepository.java +++ b/src/main/java/pro/belbix/ethparser/repositories/v0/HarvestRepository.java @@ -312,6 +312,8 @@ Page fetchPagesByVault( + "group by h.owner") List fetchUniqueAddressByNetwork(@Param("network") String network); + List findAllByOwner(String owner); + interface UserBalance { String getOwner(); diff --git a/src/main/java/pro/belbix/ethparser/service/ProfitService.java b/src/main/java/pro/belbix/ethparser/service/ProfitService.java index 6701e55f..2c5d50ab 100644 --- a/src/main/java/pro/belbix/ethparser/service/ProfitService.java +++ b/src/main/java/pro/belbix/ethparser/service/ProfitService.java @@ -38,6 +38,20 @@ public ProfitService(HarvestRepository harvestRepository, } + public Long calculateProfit(String address) { + List tx = harvestRepository.findAllByOwner(address); + Long totalProfit = 0L; + + for (HarvestDTO i: tx) { + if (i.getMethodName().equals("Withdraw")) { + totalProfit += i.getUsdAmount(); + } else { + totalProfit -= i.getUsdAmount(); + } + } + + return totalProfit; + } public Double calculationProfitForPeriod(String address, String start, String end) { diff --git a/src/main/java/pro/belbix/ethparser/service/external/CovalenthqService.java b/src/main/java/pro/belbix/ethparser/service/external/CovalenthqService.java index 9b1f24dd..ec17aa51 100644 --- a/src/main/java/pro/belbix/ethparser/service/external/CovalenthqService.java +++ b/src/main/java/pro/belbix/ethparser/service/external/CovalenthqService.java @@ -13,9 +13,9 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.springframework.web.client.RestTemplate; -import pro.belbix.ethparser.model.CovalenthqTransaction; -import pro.belbix.ethparser.model.CovalenthqTransaction.CovalenthqTransactionItems.CovalenthqTransactionItem; -import pro.belbix.ethparser.model.CovalenthqTransaction.CovalenthqTransactionItems.CovalenthqTransactionItem.CovalenthqTransactionItemLog; +import pro.belbix.ethparser.model.CovalenthqTransactionHistory; +import pro.belbix.ethparser.model.CovalenthqTransactionHistory.CovalenthqTransactionHistoryItems.CovalenthqTransactionHistoryItem; +import pro.belbix.ethparser.model.CovalenthqTransactionHistory.CovalenthqTransactionHistoryItems.CovalenthqTransactionHistoryItem.CovalenthqTransactionHistoryItemLog; import pro.belbix.ethparser.properties.ExternalProperties; import pro.belbix.ethparser.utils.UrlUtils.CovalenthqUrl; @@ -24,6 +24,7 @@ @Slf4j @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) public class CovalenthqService { + private final static int TRANSFER_LIMIT = 3; private final static String TRANSFER_LOG_NAME = "Transfer"; private final static Map CHAIN_BY_NETWORK = Map.of( @@ -50,8 +51,9 @@ public long getCreatedBlockByLastTransaction(String address, String network) { while (createdTx == null) { Thread.sleep(100); page++; - result = getTransactionByAddress(address, network, true, false, page, TRANSFER_LIMIT); - if (result == null || result.getData() == null || result.getData().getItems() == null || result.getData().getItems().isEmpty()) { + result = getTransactionByAddress(address, network, true, true, page, TRANSFER_LIMIT); + if (result == null || result.getData() == null || result.getData().getItems() == null + || result.getData().getItems().isEmpty()) { return 0; } createdTx = findCovalenthqTransactionItem(result); @@ -64,11 +66,28 @@ public long getCreatedBlockByLastTransaction(String address, String network) { } } - public CovalenthqTransaction getTransactionByAddress(String address, String network, boolean isSortAsc, boolean isFullLogs, int page, int limit) { - var url = String.format(CovalenthqUrl.TRANSACTION, externalProperties.getCovalenthq().getUrl(), convertToNetwork(network), address, isSortAsc, isFullLogs, + + public CovalenthqTransactionHistory getTransactionByAddress(String address, String network, + boolean isSortAsc, boolean isFullLogs, int page, int limit) { + var url = String.format(CovalenthqUrl.TRANSACTION_HISTORY, + externalProperties.getCovalenthq().getUrl(), convertToNetwork(network), address, isSortAsc, + isFullLogs, externalProperties.getCovalenthq().getKey(), page, limit); try { - return restTemplate.getForObject(url, CovalenthqTransaction.class); + return restTemplate.getForObject(url, CovalenthqTransactionHistory.class); + } catch (Exception e) { + log.error("Error during call {}", url, e); + throw new IllegalStateException(e); + } + } + + public CovalenthqTransactionHistory getTransactionByAddress(String address, String network, + int page, int limit, long startingBlock, long endingBlock) { + var url = String.format(CovalenthqUrl.TRANSACTION_HISTORY_WITH_BLOCK_RANGE, + externalProperties.getCovalenthq().getUrl(), convertToNetwork(network), address, + externalProperties.getCovalenthq().getKey(), page, limit, startingBlock, endingBlock); + try { + return restTemplate.getForObject(url, CovalenthqTransactionHistory.class); } catch (Exception e) { log.error("Error during call {}", url, e); throw new IllegalStateException(e); @@ -79,19 +98,20 @@ private String convertToNetwork(String network) { return Optional.ofNullable(CHAIN_BY_NETWORK.get(network)).orElse("1"); } - private CovalenthqTransactionItem findCovalenthqTransactionItem(CovalenthqTransaction result) { + private CovalenthqTransactionHistoryItem findCovalenthqTransactionItem( + CovalenthqTransactionHistory result) { return result.getData().getItems().stream() .filter(items -> items.getLogs() != null && notHasTransferInLog(items.getLogs())) .findFirst() .orElse(null); } - private boolean notHasTransferInLog(List logs) { - for (CovalenthqTransactionItemLog log : logs) { + private boolean notHasTransferInLog(List logs) { + for (CovalenthqTransactionHistoryItemLog log : logs) { if (log.getDecoded() != null && TRANSFER_LOG_NAME.equals(log.getDecoded().getName())) { return false; } } return true; } -} +} \ No newline at end of file diff --git a/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java b/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java new file mode 100644 index 00000000..2e18a5f8 --- /dev/null +++ b/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java @@ -0,0 +1,146 @@ +package pro.belbix.ethparser.service.task; + +import java.util.LinkedList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executors; +import java.util.stream.Collectors; +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; +import lombok.experimental.FieldDefaults; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.data.domain.PageRequest; +import org.springframework.data.domain.Sort; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import pro.belbix.ethparser.entity.contracts.VaultEntity; +import pro.belbix.ethparser.entity.profit.CovalenthqVaultTransaction; +import pro.belbix.ethparser.entity.profit.CovalenthqVaultTransactionType; +import pro.belbix.ethparser.model.CovalenthqTransactionHistory.CovalenthqTransactionHistoryItems.CovalenthqTransactionHistoryItem; +import pro.belbix.ethparser.model.CovalenthqTransactionHistory.CovalenthqTransactionHistoryItems.CovalenthqTransactionHistoryItem.CovalenthqTransactionHistoryItemLog; +import pro.belbix.ethparser.model.CovalenthqTransactionHistory.CovalenthqTransactionHistoryItems.CovalenthqTransactionHistoryItem.CovalenthqTransactionHistoryItemLog.CovalenthqTransactionHistoryItemLogDecode.CovalenthqTransactionHistoryItemLogDecodeParam; +import pro.belbix.ethparser.model.LogEventParam; +import pro.belbix.ethparser.repositories.covalenthq.CovalenthqVaultTransactionRepository; +import pro.belbix.ethparser.repositories.eth.VaultRepository; +import pro.belbix.ethparser.service.external.CovalenthqService; +import pro.belbix.ethparser.web3.Web3Functions; + +@Service +@RequiredArgsConstructor +@Slf4j +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) +public class CovalenthqTransactionTask { + // max block range in covalenthq + private static final int MAX_BLOCK_RANGE = 1000000; + private static final int MINUS_BLOCK = 10; + private static final int PAGINATION_SIZE = 1000000; + private static final String DEPOSIT_NAME = "Deposit"; + private static final String WITHDRAW_NAME = "Withdraw"; + + VaultRepository vaultRepository; + CovalenthqService covalenthqService; + CovalenthqVaultTransactionRepository covalenthqVaultTransactionRepository; + Web3Functions web3Functions; + + // everyday + @Scheduled(fixedRate = 1000 * 60 * 60 * 24) + public void start() { + log.info("Begin parse vault tx"); + var executor = Executors.newFixedThreadPool(10); + + var futures = vaultRepository.findAll().stream() + .map(i -> CompletableFuture.runAsync(() -> getVaultTransaction(i), executor)) + .collect(Collectors.toList()); + + } + + // TODO add more logs, check on null, catch exceptions + private void getVaultTransaction(VaultEntity vault) { + log.info("Run getVaultTransaction for {}", vault); + var contract = vault.getContract(); + var lastTx = covalenthqVaultTransactionRepository.findAllByNetworkAndContractAddress(contract.getNetwork(), contract.getAddress(), + PageRequest.of(0, 1, Sort.by("blockHeight").ascending())); + + var startingBlock = 0L; + + if (!lastTx.isEmpty()) { + startingBlock = Math.max(lastTx.get(0).getBlock() - MINUS_BLOCK, 0); + } + + var currentBlock = web3Functions.fetchCurrentBlock(contract.getNetwork()); + var endingBlock = startingBlock + MAX_BLOCK_RANGE; + var result = fetchTransactions(contract.getAddress(), contract.getNetwork(), startingBlock, endingBlock); + + while (currentBlock.longValue() >= endingBlock) { + // TODO Maybe need to add one more block, because can find the same tx + startingBlock = endingBlock; + endingBlock += MAX_BLOCK_RANGE; + result.addAll( + fetchTransactions(contract.getAddress(), contract.getNetwork(), startingBlock, endingBlock)); + } + + var transactions = result.stream() + .filter(this::isDepositOrWithdraw) + .map(tx -> tx.getLogs().stream() + .map(log -> toCovalenthqVaultTransaction(log, contract.getNetwork(), contract.getAddress())) + .collect(Collectors.toList()) + ) + .flatMap(List::stream) + .collect(Collectors.toList()); + + covalenthqVaultTransactionRepository.saveAll(transactions); + } + + private List fetchTransactions(String address, String network, long startingBlock, long endingBlock) { + var page = 0; + var transaction = covalenthqService.getTransactionByAddress(address, network, page, PAGINATION_SIZE, startingBlock, endingBlock); + var result = new LinkedList(transaction.getData().getItems()); + var hasMore = transaction.getData().getPagination().isHasMore(); + + while (hasMore) { + page++; + transaction = covalenthqService.getTransactionByAddress(address, network, page, PAGINATION_SIZE, startingBlock, endingBlock); + hasMore = transaction.getData().getPagination().isHasMore(); + result.addAll(transaction.getData().getItems()); + } + return result; + } + + private boolean isDepositOrWithdraw(CovalenthqTransactionHistoryItem item) { + return item.getLogs().stream() + .anyMatch(i -> + WITHDRAW_NAME.equalsIgnoreCase(i.getDecoded().getName()) + || DEPOSIT_NAME.equalsIgnoreCase(i.getDecoded().getName())); + } + + private CovalenthqVaultTransaction toCovalenthqVaultTransaction( + CovalenthqTransactionHistoryItemLog item, String network, String contractAddress) { + var transaction = new CovalenthqVaultTransaction(); + var type = toCovalenthqVaultTransactionType(item); + + transaction.setNetwork(network); + transaction.setBlock(item.getBlockHeight()); + transaction.setTransactionHash(item.getTransactionHash()); + transaction.setContractDecimal(item.getContractDecimal()); + transaction.setContractAddress(contractAddress); + transaction.setType(type); + transaction.setOwnerAddress(getParamValue(item, type.equals(CovalenthqVaultTransactionType.DEPOSIT) ? LogEventParam.DEPOSIT_FROM : LogEventParam.WITHDRAW_TO)); + transaction.setValue(Long.parseLong(getParamValue(item, type.equals(CovalenthqVaultTransactionType.DEPOSIT) ? LogEventParam.DEPOSIT_VALUE : LogEventParam.WITHDRAW_VALUE))); + transaction.setSignedAt(transaction.getSignedAt()); + + return transaction; + } + + private CovalenthqVaultTransactionType toCovalenthqVaultTransactionType(CovalenthqTransactionHistoryItemLog item) { + return WITHDRAW_NAME.equalsIgnoreCase(item.getDecoded().getName()) ? CovalenthqVaultTransactionType.WITHDRAW : CovalenthqVaultTransactionType.DEPOSIT; + } + + private String getParamValue(CovalenthqTransactionHistoryItemLog item, LogEventParam param) { + return item.getDecoded().getParams().stream() + .filter(i -> param.value.equals(i.getName())) + .map(CovalenthqTransactionHistoryItemLogDecodeParam::getValue) + .findFirst() + .orElse(StringUtils.EMPTY); + } +} diff --git a/src/main/java/pro/belbix/ethparser/service/HarvestPoolInfoService.java b/src/main/java/pro/belbix/ethparser/service/task/HarvestPoolInfoTask.java similarity index 98% rename from src/main/java/pro/belbix/ethparser/service/HarvestPoolInfoService.java rename to src/main/java/pro/belbix/ethparser/service/task/HarvestPoolInfoTask.java index 7051fea0..202e6b76 100644 --- a/src/main/java/pro/belbix/ethparser/service/HarvestPoolInfoService.java +++ b/src/main/java/pro/belbix/ethparser/service/task/HarvestPoolInfoTask.java @@ -1,4 +1,4 @@ -package pro.belbix.ethparser.service; +package pro.belbix.ethparser.service.task; import static pro.belbix.ethparser.service.AbiProviderService.BSC_NETWORK; import static pro.belbix.ethparser.service.AbiProviderService.ETH_NETWORK; @@ -32,7 +32,7 @@ @RequiredArgsConstructor @Slf4j @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) -public class HarvestPoolInfoService { +public class HarvestPoolInfoTask { private static final Long INCREASE_BLOCK_WEIGHT = 1000L; HarvestService harvestService; diff --git a/src/main/java/pro/belbix/ethparser/service/HarvestVaultInfoService.java b/src/main/java/pro/belbix/ethparser/service/task/HarvestVaultInfoTask.java similarity index 98% rename from src/main/java/pro/belbix/ethparser/service/HarvestVaultInfoService.java rename to src/main/java/pro/belbix/ethparser/service/task/HarvestVaultInfoTask.java index 2de39a4f..cf99d2f9 100644 --- a/src/main/java/pro/belbix/ethparser/service/HarvestVaultInfoService.java +++ b/src/main/java/pro/belbix/ethparser/service/task/HarvestVaultInfoTask.java @@ -1,4 +1,4 @@ -package pro.belbix.ethparser.service; +package pro.belbix.ethparser.service.task; import static pro.belbix.ethparser.service.AbiProviderService.BSC_NETWORK; import static pro.belbix.ethparser.service.AbiProviderService.ETH_NETWORK; @@ -33,7 +33,7 @@ @RequiredArgsConstructor @Slf4j @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) -public class HarvestVaultInfoService { +public class HarvestVaultInfoTask { private static final Long INCREASE_BLOCK_WEIGHT = 1000L; HarvestService harvestService; diff --git a/src/main/java/pro/belbix/ethparser/utils/UrlUtils.java b/src/main/java/pro/belbix/ethparser/utils/UrlUtils.java index 48b1001e..d9861804 100644 --- a/src/main/java/pro/belbix/ethparser/utils/UrlUtils.java +++ b/src/main/java/pro/belbix/ethparser/utils/UrlUtils.java @@ -3,14 +3,33 @@ public class UrlUtils { public interface CovalenthqUrl { - String TRANSACTION = "%s%s/address/%s/transactions_v2/?" - + "quote-currency=USD" - + "&format=JSON" - + "&block-signed-at-asc=%s" - + "&no-logs=%s" - + "&key=%s" - + "&page-number=%s" - + "&page-size=%s"; + String TRANSACTION_HISTORY = "%s%s/address/%s/transactions_v2/?" + + "quote-currency=USD&" + + "format=JSON&" + + "block-signed-at-asc=%s&" + + "no-logs=%s&" + + "key=%s&" + + "page-number=%s&" + + "page-size=%s&"; + // TODO move params to restTemplate + String TRANSACTION_HISTORY_WITH_BLOCK_RANGE = "%s%s/address/%s/transactions_v2/?" + + "quote-currency=USD&" + + "format=JSON&" + + "block-signed-at-asc=true&" + + "no-logs=false&" + + "key=%s&" + + "page-number=%s&" + + "page-size=%s&" + + "starting-block=%s&" + + "ending-block=%s&"; + + String TRANSACTION_BY_CONTRACT_ID = "%s%s/address/%s/transfers_v2/?" + + "contract-address=%s&" + + "key=%s&" + + "page-number=%s&" + + "page-size=%s&" + + "starting-block=%s&" + + "ending-block=%s"; } public interface HarvestUrl { From 42cca8369f21406b7bd218b958829d539521055e Mon Sep 17 00:00:00 2001 From: alex Date: Mon, 28 Feb 2022 17:19:02 +0300 Subject: [PATCH 15/65] Resolve problem with covalnthq API --- pom.xml | 10 + scripts/run_app/run.sh | 2 +- scripts/run_app/run_config.yml | 4 +- scripts/sql/schema.sql | 6 +- .../java/pro/belbix/ethparser/AppConfig.java | 2 + .../java/pro/belbix/ethparser/AppStarter.java | 25 +- .../profit/CovalenthqVaultTransaction.java | 8 +- .../CovalenthqVaultTransactionType.java | 20 +- .../model/CovalenthqTransactionHistory.java | 20 +- .../CovalenthqVaultTransactionRepository.java | 1 + .../service/external/CovalenthqService.java | 17 +- .../task/CovalenthqTransactionTask.java | 251 +++++++++++++----- .../belbix/ethparser/web3/MethodDecoder.java | 6 +- .../belbix/ethparser/web3/SimpleDecoder.java | 2 + .../web3/erc20/parser/TransferParser.java | 1 - 15 files changed, 254 insertions(+), 121 deletions(-) mode change 100644 => 100755 scripts/run_app/run.sh diff --git a/pom.xml b/pom.xml index 4304882d..650977bf 100644 --- a/pom.xml +++ b/pom.xml @@ -42,6 +42,16 @@ org.springframework.boot spring-boot-starter-data-rest + + org.springframework.retry + spring-retry + 1.1.5.RELEASE + + + org.springframework + spring-aspects + 5.2.8.RELEASE + org.springframework.boot spring-boot-starter-test diff --git a/scripts/run_app/run.sh b/scripts/run_app/run.sh old mode 100644 new mode 100755 index 852492e2..911443ae --- a/scripts/run_app/run.sh +++ b/scripts/run_app/run.sh @@ -4,4 +4,4 @@ rm ethparser.jar mvn install -Dmaven.test.skip=true -f ./../../pom.xml cp -R ./../../dist/lib/. ./lib cp ./../../dist/ethparser.jar ./ethparser.jar -java -Xmx1g -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=5005 -Dspring.config.location=./../application.yml,run_config.yml -cp ethparser.jar pro.belbix.ethparser.Application +java -Xmx4g -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=5005 -Dspring.config.location=./../application.yml,run_config.yml -cp ethparser.jar pro.belbix.ethparser.Application diff --git a/scripts/run_app/run_config.yml b/scripts/run_app/run_config.yml index 5c4a556b..3e4aea06 100644 --- a/scripts/run_app/run_config.yml +++ b/scripts/run_app/run_config.yml @@ -1,7 +1,7 @@ ethparser: # overrideDuplicates: true - stopOnParseError: true - onlyApi: true + stopOnParseError: false + onlyApi: false # eth: # onlyParse: true # startLogBlock: 10765094 diff --git a/scripts/sql/schema.sql b/scripts/sql/schema.sql index 9a5da908..99d3efde 100644 --- a/scripts/sql/schema.sql +++ b/scripts/sql/schema.sql @@ -1111,12 +1111,12 @@ create table covalenthq_vault_tx id bigserial primary key, network varchar(255), - block numeric(19, 2), + block numeric(19, 0), transaction_hash varchar(255), - contract_decimal numeric(19, 2), + contract_decimal numeric(19, 0), contract_address varchar(255), owner_address varchar(255), - value numeric(19, 2), + value numeric(60, 0), signed_at timestamp, type varchar(255) ); diff --git a/src/main/java/pro/belbix/ethparser/AppConfig.java b/src/main/java/pro/belbix/ethparser/AppConfig.java index 693a293b..06287a08 100644 --- a/src/main/java/pro/belbix/ethparser/AppConfig.java +++ b/src/main/java/pro/belbix/ethparser/AppConfig.java @@ -4,6 +4,7 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.messaging.simp.config.MessageBrokerRegistry; +import org.springframework.retry.annotation.EnableRetry; import org.springframework.scheduling.annotation.EnableScheduling; import org.springframework.web.client.RestTemplate; import org.springframework.web.socket.config.annotation.EnableWebSocketMessageBroker; @@ -24,6 +25,7 @@ ExternalProperties.class }) @EnableScheduling +@EnableRetry public class AppConfig { @Configuration diff --git a/src/main/java/pro/belbix/ethparser/AppStarter.java b/src/main/java/pro/belbix/ethparser/AppStarter.java index 17ee01cf..2f900e74 100644 --- a/src/main/java/pro/belbix/ethparser/AppStarter.java +++ b/src/main/java/pro/belbix/ethparser/AppStarter.java @@ -1,13 +1,10 @@ package pro.belbix.ethparser; import static pro.belbix.ethparser.ws.WsService.BANCOR_TRANSACTIONS_TOPIC_NAME; -import static pro.belbix.ethparser.ws.WsService.DEPLOYER_TRANSACTIONS_TOPIC_NAME; import static pro.belbix.ethparser.ws.WsService.HARDWORK_TOPIC_NAME; import static pro.belbix.ethparser.ws.WsService.HARVEST_TRANSACTIONS_TOPIC_NAME; import static pro.belbix.ethparser.ws.WsService.IMPORTANT_EVENTS_TOPIC_NAME; import static pro.belbix.ethparser.ws.WsService.PRICES_TOPIC_NAME; -import static pro.belbix.ethparser.ws.WsService.REWARDS_TOPIC_NAME; -import static pro.belbix.ethparser.ws.WsService.TRANSFERS_TOPIC_NAME; import static pro.belbix.ethparser.ws.WsService.UNI_TRANSACTIONS_TOPIC_NAME; import java.util.Arrays; @@ -102,17 +99,17 @@ public void start() { if (conf.isTestWs()) { startFakeDataForWebSocket(ws, conf.getTestWsRate()); } else { - startParse(bancorPriceParser, ws, BANCOR_TRANSACTIONS_TOPIC_NAME, true); - startParse(uniswapLpLogParser, ws, UNI_TRANSACTIONS_TOPIC_NAME, true); - startParse(vaultActionsParser, ws, HARVEST_TRANSACTIONS_TOPIC_NAME, true); - startParse(hardWorkParser, ws, HARDWORK_TOPIC_NAME, true); - startParse(rewardParser, ws, REWARDS_TOPIC_NAME, true); - startParse(importantEventsParser, ws, IMPORTANT_EVENTS_TOPIC_NAME, true); - startParse(uniToHarvestConverter, ws, HARVEST_TRANSACTIONS_TOPIC_NAME, true); - startParse(transferParser, ws, TRANSFERS_TOPIC_NAME, true); - startParse(priceLogParser, ws, PRICES_TOPIC_NAME, true); - startParse(deployerTransactionsParser, ws, - DEPLOYER_TRANSACTIONS_TOPIC_NAME, false); +// startParse(bancorPriceParser, ws, BANCOR_TRANSACTIONS_TOPIC_NAME, true); +// startParse(uniswapLpLogParser, ws, UNI_TRANSACTIONS_TOPIC_NAME, true); +// startParse(vaultActionsParser, ws, HARVEST_TRANSACTIONS_TOPIC_NAME, true); +// startParse(hardWorkParser, ws, HARDWORK_TOPIC_NAME, true); +// startParse(rewardParser, ws, REWARDS_TOPIC_NAME, true); +// startParse(importantEventsParser, ws, IMPORTANT_EVENTS_TOPIC_NAME, true); +// startParse(uniToHarvestConverter, ws, HARVEST_TRANSACTIONS_TOPIC_NAME, true); +// startParse(transferParser, ws, TRANSFERS_TOPIC_NAME, true); +// startParse(priceLogParser, ws, PRICES_TOPIC_NAME, true); +// startParse(deployerTransactionsParser, ws, +// DEPLOYER_TRANSACTIONS_TOPIC_NAME, false); startParseBlocks(); } } diff --git a/src/main/java/pro/belbix/ethparser/entity/profit/CovalenthqVaultTransaction.java b/src/main/java/pro/belbix/ethparser/entity/profit/CovalenthqVaultTransaction.java index cd74dac5..7e976505 100644 --- a/src/main/java/pro/belbix/ethparser/entity/profit/CovalenthqVaultTransaction.java +++ b/src/main/java/pro/belbix/ethparser/entity/profit/CovalenthqVaultTransaction.java @@ -1,10 +1,9 @@ package pro.belbix.ethparser.entity.profit; +import java.math.BigDecimal; import java.time.LocalDateTime; import javax.persistence.Entity; -import javax.persistence.EnumType; -import javax.persistence.Enumerated; import javax.persistence.GeneratedValue; import javax.persistence.GenerationType; import javax.persistence.Id; @@ -28,8 +27,7 @@ public class CovalenthqVaultTransaction { int contractDecimal; String contractAddress; String ownerAddress; - long value; + BigDecimal value; LocalDateTime signedAt; - @Enumerated(EnumType.STRING) - CovalenthqVaultTransactionType type; + String type; } diff --git a/src/main/java/pro/belbix/ethparser/entity/profit/CovalenthqVaultTransactionType.java b/src/main/java/pro/belbix/ethparser/entity/profit/CovalenthqVaultTransactionType.java index 27c941f0..524f67ae 100644 --- a/src/main/java/pro/belbix/ethparser/entity/profit/CovalenthqVaultTransactionType.java +++ b/src/main/java/pro/belbix/ethparser/entity/profit/CovalenthqVaultTransactionType.java @@ -1,6 +1,22 @@ package pro.belbix.ethparser.entity.profit; +import lombok.AccessLevel; +import lombok.experimental.FieldDefaults; + +@FieldDefaults(makeFinal = true, level = AccessLevel.PUBLIC) public enum CovalenthqVaultTransactionType { - WITHDRAW, - DEPOSIT + WITHDRAW_UNI("account", "writeAmount", "Withdraw"), + WITHDRAW("provider", "value", "Withdraw"), + DEPOSIT_UNI("user", "amount", "Deposit"), + DEPOSIT("dst", "wad", "Deposit"); + + CovalenthqVaultTransactionType(String address, String value, String type) { + this.address = address; + this.value = value; + this.type = type; + } + + String address; + String value; + String type; } diff --git a/src/main/java/pro/belbix/ethparser/model/CovalenthqTransactionHistory.java b/src/main/java/pro/belbix/ethparser/model/CovalenthqTransactionHistory.java index 7430614d..9ff619e5 100644 --- a/src/main/java/pro/belbix/ethparser/model/CovalenthqTransactionHistory.java +++ b/src/main/java/pro/belbix/ethparser/model/CovalenthqTransactionHistory.java @@ -35,24 +35,12 @@ public static class CovalenthqTransactionHistoryItemLog { private long blockHeight; @JsonProperty("tx_hash") private String transactionHash; - private CovalenthqTransactionHistoryItemLogDecode decoded; @JsonProperty("sender_contract_decimals") private int contractDecimal; - - @Data - public static class CovalenthqTransactionHistoryItemLogDecode { - private String name; - private List params; - - @Data - public static class CovalenthqTransactionHistoryItemLogDecodeParam { - private String name; - private String type; - private boolean indexed; - private boolean decoded; - private String value; - } - } + @JsonProperty(value = "raw_log_topics") + private List topics; + @JsonProperty("raw_log_data") + private String data; } } diff --git a/src/main/java/pro/belbix/ethparser/repositories/covalenthq/CovalenthqVaultTransactionRepository.java b/src/main/java/pro/belbix/ethparser/repositories/covalenthq/CovalenthqVaultTransactionRepository.java index 966dcf2d..6947f52a 100644 --- a/src/main/java/pro/belbix/ethparser/repositories/covalenthq/CovalenthqVaultTransactionRepository.java +++ b/src/main/java/pro/belbix/ethparser/repositories/covalenthq/CovalenthqVaultTransactionRepository.java @@ -7,4 +7,5 @@ public interface CovalenthqVaultTransactionRepository extends JpaRepository { List findAllByNetworkAndContractAddress(String network, String contractAddress, Pageable pageable); + List findAllByTransactionHashIn(List transactionHashes); } diff --git a/src/main/java/pro/belbix/ethparser/service/external/CovalenthqService.java b/src/main/java/pro/belbix/ethparser/service/external/CovalenthqService.java index ec17aa51..e8a42f92 100644 --- a/src/main/java/pro/belbix/ethparser/service/external/CovalenthqService.java +++ b/src/main/java/pro/belbix/ethparser/service/external/CovalenthqService.java @@ -11,6 +11,8 @@ import lombok.RequiredArgsConstructor; import lombok.experimental.FieldDefaults; import lombok.extern.slf4j.Slf4j; +import org.springframework.retry.annotation.Backoff; +import org.springframework.retry.annotation.Retryable; import org.springframework.stereotype.Service; import org.springframework.web.client.RestTemplate; import pro.belbix.ethparser.model.CovalenthqTransactionHistory; @@ -81,6 +83,8 @@ public CovalenthqTransactionHistory getTransactionByAddress(String address, Stri } } + // if 524 response code try to resize page count + @Retryable(value = Exception.class, maxAttempts = 4, backoff = @Backoff(delay = 1000)) public CovalenthqTransactionHistory getTransactionByAddress(String address, String network, int page, int limit, long startingBlock, long endingBlock) { var url = String.format(CovalenthqUrl.TRANSACTION_HISTORY_WITH_BLOCK_RANGE, @@ -107,11 +111,12 @@ private CovalenthqTransactionHistoryItem findCovalenthqTransactionItem( } private boolean notHasTransferInLog(List logs) { - for (CovalenthqTransactionHistoryItemLog log : logs) { - if (log.getDecoded() != null && TRANSFER_LOG_NAME.equals(log.getDecoded().getName())) { - return false; - } - } +// for (CovalenthqTransactionHistoryItemLog log : logs) { +// if (log.getDecoded() != null && TRANSFER_LOG_NAME.equals(log.getDecoded().getName())) { +// return false; +// } +// } return true; } -} \ No newline at end of file +} + diff --git a/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java b/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java index 2e18a5f8..669e4360 100644 --- a/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java +++ b/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java @@ -1,7 +1,19 @@ package pro.belbix.ethparser.service.task; +import static pro.belbix.ethparser.entity.profit.CovalenthqVaultTransactionType.DEPOSIT; +import static pro.belbix.ethparser.entity.profit.CovalenthqVaultTransactionType.DEPOSIT_UNI; +import static pro.belbix.ethparser.entity.profit.CovalenthqVaultTransactionType.WITHDRAW; +import static pro.belbix.ethparser.entity.profit.CovalenthqVaultTransactionType.WITHDRAW_UNI; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.Collection; +import java.util.HashSet; import java.util.LinkedList; import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executors; import java.util.stream.Collectors; @@ -14,16 +26,16 @@ import org.springframework.data.domain.Sort; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; +import org.web3j.protocol.core.methods.response.Log; import pro.belbix.ethparser.entity.contracts.VaultEntity; import pro.belbix.ethparser.entity.profit.CovalenthqVaultTransaction; import pro.belbix.ethparser.entity.profit.CovalenthqVaultTransactionType; import pro.belbix.ethparser.model.CovalenthqTransactionHistory.CovalenthqTransactionHistoryItems.CovalenthqTransactionHistoryItem; import pro.belbix.ethparser.model.CovalenthqTransactionHistory.CovalenthqTransactionHistoryItems.CovalenthqTransactionHistoryItem.CovalenthqTransactionHistoryItemLog; -import pro.belbix.ethparser.model.CovalenthqTransactionHistory.CovalenthqTransactionHistoryItems.CovalenthqTransactionHistoryItem.CovalenthqTransactionHistoryItemLog.CovalenthqTransactionHistoryItemLogDecode.CovalenthqTransactionHistoryItemLogDecodeParam; -import pro.belbix.ethparser.model.LogEventParam; import pro.belbix.ethparser.repositories.covalenthq.CovalenthqVaultTransactionRepository; import pro.belbix.ethparser.repositories.eth.VaultRepository; import pro.belbix.ethparser.service.external.CovalenthqService; +import pro.belbix.ethparser.web3.SimpleDecoder; import pro.belbix.ethparser.web3.Web3Functions; @Service @@ -35,9 +47,15 @@ public class CovalenthqTransactionTask { private static final int MAX_BLOCK_RANGE = 1000000; private static final int MINUS_BLOCK = 10; private static final int PAGINATION_SIZE = 1000000; - private static final String DEPOSIT_NAME = "Deposit"; - private static final String WITHDRAW_NAME = "Withdraw"; + private static final int PAGINATION_SIZE_FOR_A_LOT_OF_DATA = 1000; + private static final Map LOG_EVENTS = Map.of( + "0xf279e6a1f5e320cca91135676d9cb6e44ca8a08c0b88342bcdb1144f6511b568", WITHDRAW_UNI, + "0x884edad9ce6fa2440d8a54cc123490eb96d2768479d49ff9c7366125a9424364", WITHDRAW, + "0x90890809c654f11d6e72a28fa60149770a0d11ec6c92319d6ceb2bb0a4ea1a15", DEPOSIT_UNI, + "0xe1fffcc4923d04b559f4d29a8bfc6cda04eb5b0d3c460751c2402c5c5cc9109c", DEPOSIT + ); + SimpleDecoder simpleDecoder; VaultRepository vaultRepository; CovalenthqService covalenthqService; CovalenthqVaultTransactionRepository covalenthqVaultTransactionRepository; @@ -47,7 +65,7 @@ public class CovalenthqTransactionTask { @Scheduled(fixedRate = 1000 * 60 * 60 * 24) public void start() { log.info("Begin parse vault tx"); - var executor = Executors.newFixedThreadPool(10); + var executor = Executors.newFixedThreadPool(20); var futures = vaultRepository.findAll().stream() .map(i -> CompletableFuture.runAsync(() -> getVaultTransaction(i), executor)) @@ -57,90 +75,183 @@ public void start() { // TODO add more logs, check on null, catch exceptions private void getVaultTransaction(VaultEntity vault) { - log.info("Run getVaultTransaction for {}", vault); - var contract = vault.getContract(); - var lastTx = covalenthqVaultTransactionRepository.findAllByNetworkAndContractAddress(contract.getNetwork(), contract.getAddress(), - PageRequest.of(0, 1, Sort.by("blockHeight").ascending())); + try { + log.info("Run getVaultTransaction for {}", vault); + var contract = vault.getContract(); + var lastTx = covalenthqVaultTransactionRepository.findAllByNetworkAndContractAddress(contract.getNetwork(), contract.getAddress(), + PageRequest.of(0, 1, Sort.by("block").ascending())); - var startingBlock = 0L; + var startingBlock = 0L; - if (!lastTx.isEmpty()) { - startingBlock = Math.max(lastTx.get(0).getBlock() - MINUS_BLOCK, 0); - } + if (!lastTx.isEmpty()) { + startingBlock = Math.max(lastTx.get(0).getBlock() - MINUS_BLOCK, 0); + } - var currentBlock = web3Functions.fetchCurrentBlock(contract.getNetwork()); - var endingBlock = startingBlock + MAX_BLOCK_RANGE; - var result = fetchTransactions(contract.getAddress(), contract.getNetwork(), startingBlock, endingBlock); + var currentBlock = web3Functions.fetchCurrentBlock(contract.getNetwork()); + var endingBlock = startingBlock + MAX_BLOCK_RANGE; + var result = fetchTransactions(contract.getAddress(), contract.getNetwork(), startingBlock, endingBlock); - while (currentBlock.longValue() >= endingBlock) { - // TODO Maybe need to add one more block, because can find the same tx - startingBlock = endingBlock; - endingBlock += MAX_BLOCK_RANGE; - result.addAll( - fetchTransactions(contract.getAddress(), contract.getNetwork(), startingBlock, endingBlock)); - } + while (currentBlock.longValue() >= endingBlock) { + // TODO Maybe need to add one more block, because can find the same tx + startingBlock = endingBlock; + endingBlock += MAX_BLOCK_RANGE; + result.addAll( + fetchTransactions(contract.getAddress(), contract.getNetwork(), startingBlock, endingBlock)); + } - var transactions = result.stream() - .filter(this::isDepositOrWithdraw) - .map(tx -> tx.getLogs().stream() - .map(log -> toCovalenthqVaultTransaction(log, contract.getNetwork(), contract.getAddress())) - .collect(Collectors.toList()) - ) - .flatMap(List::stream) - .collect(Collectors.toList()); + var transactions = result.stream() + .map(CovalenthqTransactionHistoryItem::getLogs) + .flatMap(Collection::stream) + .filter(this::isDepositOrWithdraw) + .map(log -> toCovalenthqVaultTransaction(log, contract.getNetwork(), contract.getAddress())) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + + var transactionWithoutDuplicate = new HashSet<>(transactions); + var transactionInDb = covalenthqVaultTransactionRepository.findAllByTransactionHashIn( + transactionWithoutDuplicate.stream() + .map(CovalenthqVaultTransaction::getTransactionHash) + .collect(Collectors.toList())); + transactions = transactionWithoutDuplicate.stream() + .filter(i1 -> + transactionInDb.stream().noneMatch(i2 -> + i1.getNetwork().equals(i2.getNetwork()) + && i1.getTransactionHash().equals(i2.getTransactionHash()) + && i1.getContractAddress().equals(i2.getContractAddress()) + && i1.getOwnerAddress().equals(i2.getOwnerAddress()) + ) + ) + .collect(Collectors.toList()); - covalenthqVaultTransactionRepository.saveAll(transactions); + log.debug("Save list covalenthq tx: {}", transactionInDb); + covalenthqVaultTransactionRepository.saveAll(transactions); + } catch (Exception e) { + log.error("Get error getVaultTransaction", e); + } } private List fetchTransactions(String address, String network, long startingBlock, long endingBlock) { - var page = 0; - var transaction = covalenthqService.getTransactionByAddress(address, network, page, PAGINATION_SIZE, startingBlock, endingBlock); - var result = new LinkedList(transaction.getData().getItems()); - var hasMore = transaction.getData().getPagination().isHasMore(); - - while (hasMore) { - page++; - transaction = covalenthqService.getTransactionByAddress(address, network, page, PAGINATION_SIZE, startingBlock, endingBlock); - hasMore = transaction.getData().getPagination().isHasMore(); - result.addAll(transaction.getData().getItems()); + return fetchTransactions(address, network, startingBlock, endingBlock, PAGINATION_SIZE, false); + } + + private List fetchTransactions(String address, String network, long startingBlock, long endingBlock, int limit, boolean isAfterException) { + try { + var page = 0; + var transaction = covalenthqService.getTransactionByAddress(address, network, page, limit, startingBlock, endingBlock); + var result = new LinkedList(transaction.getData().getItems()); + var hasMore = transaction.getData().getPagination().isHasMore(); + + while (hasMore) { + page++; + transaction = covalenthqService.getTransactionByAddress(address, network, page, limit, startingBlock, endingBlock); + hasMore = transaction.getData().getPagination().isHasMore(); + result.addAll(transaction.getData().getItems()); + } + return result; + } catch (IllegalStateException e) { + log.debug("Get a lot of data {} {} on block {} - {}", address, network, startingBlock, endingBlock); + if (isAfterException) { + throw e; + } + return fetchTransactions(address, network, startingBlock, endingBlock, PAGINATION_SIZE_FOR_A_LOT_OF_DATA, true); } - return result; } - private boolean isDepositOrWithdraw(CovalenthqTransactionHistoryItem item) { - return item.getLogs().stream() - .anyMatch(i -> - WITHDRAW_NAME.equalsIgnoreCase(i.getDecoded().getName()) - || DEPOSIT_NAME.equalsIgnoreCase(i.getDecoded().getName())); + private boolean isDepositOrWithdraw(CovalenthqTransactionHistoryItemLog log) { + return log.getTopics() != null && !log.getTopics().isEmpty() && LOG_EVENTS.get(log.getTopics().get(0)) != null; } private CovalenthqVaultTransaction toCovalenthqVaultTransaction( CovalenthqTransactionHistoryItemLog item, String network, String contractAddress) { - var transaction = new CovalenthqVaultTransaction(); - var type = toCovalenthqVaultTransactionType(item); - - transaction.setNetwork(network); - transaction.setBlock(item.getBlockHeight()); - transaction.setTransactionHash(item.getTransactionHash()); - transaction.setContractDecimal(item.getContractDecimal()); - transaction.setContractAddress(contractAddress); - transaction.setType(type); - transaction.setOwnerAddress(getParamValue(item, type.equals(CovalenthqVaultTransactionType.DEPOSIT) ? LogEventParam.DEPOSIT_FROM : LogEventParam.WITHDRAW_TO)); - transaction.setValue(Long.parseLong(getParamValue(item, type.equals(CovalenthqVaultTransactionType.DEPOSIT) ? LogEventParam.DEPOSIT_VALUE : LogEventParam.WITHDRAW_VALUE))); - transaction.setSignedAt(transaction.getSignedAt()); - - return transaction; + try { + var transaction = new CovalenthqVaultTransaction(); + var covalenthqType = Optional.ofNullable(toCovalenthqVaultTransactionType(item)) + .orElseThrow(() -> { + log.error("Can not find log event"); + throw new IllegalStateException(); + }); + var value = getParamValue(item, covalenthqType.value); + + transaction.setNetwork(network); + transaction.setBlock(item.getBlockHeight()); + transaction.setTransactionHash(item.getTransactionHash()); + transaction.setContractDecimal(item.getContractDecimal()); + transaction.setContractAddress(contractAddress); + transaction.setType(covalenthqType.type); + transaction.setOwnerAddress(getParamValue(item, covalenthqType.address)); + transaction.setValue( + StringUtils.isEmpty(value) ? BigDecimal.ZERO : new BigDecimal(value) + ); + transaction.setSignedAt(transaction.getSignedAt()); + + return transaction; + } catch (Exception e) { + log.error("Can not parse covalenthq log: {}", item, e); + return null; + } } - private CovalenthqVaultTransactionType toCovalenthqVaultTransactionType(CovalenthqTransactionHistoryItemLog item) { - return WITHDRAW_NAME.equalsIgnoreCase(item.getDecoded().getName()) ? CovalenthqVaultTransactionType.WITHDRAW : CovalenthqVaultTransactionType.DEPOSIT; + private CovalenthqVaultTransactionType toCovalenthqVaultTransactionType(CovalenthqTransactionHistoryItemLog log) { + if (log.getTopics() != null && !log.getTopics().isEmpty()) { + return LOG_EVENTS.get(log.getTopics().get(0)); + } + + return null; + } + + private String getParamValue(CovalenthqTransactionHistoryItemLog item, String param) { + return getUniParamValue(item, param); + +// if (item == null || item.getDecoded() == null) { +// return StringUtils.EMPTY; +// } +// +// if (item.getDecoded().getParams() == null) { +// return getUniParamValue(item, param); +// } +// +// return item.getDecoded().getParams().stream() +// .filter(i -> param.equals(i.getName()) && i.getValue() instanceof String) +// .map(i -> (String)i.getValue()) +// .findFirst() +// .orElse(StringUtils.EMPTY); } - private String getParamValue(CovalenthqTransactionHistoryItemLog item, LogEventParam param) { - return item.getDecoded().getParams().stream() - .filter(i -> param.value.equals(i.getName())) - .map(CovalenthqTransactionHistoryItemLogDecodeParam::getValue) - .findFirst() - .orElse(StringUtils.EMPTY); + private String getUniParamValue(CovalenthqTransactionHistoryItemLog item, String param) { + var ethLog = new Log(); + ethLog.setTopics(item.getTopics()); + ethLog.setData(item.getData()); + + var result = simpleDecoder.decodeEthLog(ethLog) + .orElseThrow(); + + var indexParam = -1; + var castToString = false; + + switch (param) { + case "account": + case "user": + case "dst": + case "provider": + indexParam = 0; + castToString = true; + break; + case "wad": + case "value": + indexParam = 1; + break; + case "amount" : + case "writeAmount": + indexParam = 2; + break; + } + if (result.isEmpty() || indexParam == -1 || indexParam >= result.size()) { + log.error("Unknown param or can not parse data"); + throw new IllegalStateException(); + } + + return castToString + ? (String) result.get(indexParam).getValue() + : ((BigInteger) result.get(indexParam).getValue()).toString(); } } diff --git a/src/main/java/pro/belbix/ethparser/web3/MethodDecoder.java b/src/main/java/pro/belbix/ethparser/web3/MethodDecoder.java index 0f64936d..0cff41a9 100644 --- a/src/main/java/pro/belbix/ethparser/web3/MethodDecoder.java +++ b/src/main/java/pro/belbix/ethparser/web3/MethodDecoder.java @@ -88,7 +88,11 @@ public static List extractLogIndexedValues( } List> indexedParameters = getIndexedParameters(parameters); for (int i = 0; i < indexedParameters.size(); i++) { - String topic = topics.get(i + 1); + var index = topics.size() > i + 1 + ? i + 1 + : i; + + String topic = topics.get(index); Type value = decodeIndexedValue(topic, indexedParameters.get(i)); indexedValues.add(value); } diff --git a/src/main/java/pro/belbix/ethparser/web3/SimpleDecoder.java b/src/main/java/pro/belbix/ethparser/web3/SimpleDecoder.java index ec9b9b1c..95784310 100644 --- a/src/main/java/pro/belbix/ethparser/web3/SimpleDecoder.java +++ b/src/main/java/pro/belbix/ethparser/web3/SimpleDecoder.java @@ -4,12 +4,14 @@ import java.util.Optional; import java.util.TreeMap; import java.util.function.Predicate; +import org.springframework.stereotype.Component; import org.web3j.abi.datatypes.Type; import org.web3j.protocol.core.methods.response.EthLog.LogResult; import org.web3j.protocol.core.methods.response.Log; import org.web3j.protocol.core.methods.response.Transaction; import pro.belbix.ethparser.model.tx.EthTransactionI; +@Component public class SimpleDecoder extends MethodDecoder { public Optional> decodeEthLog(Log ethLog) { diff --git a/src/main/java/pro/belbix/ethparser/web3/erc20/parser/TransferParser.java b/src/main/java/pro/belbix/ethparser/web3/erc20/parser/TransferParser.java index 99619e98..454e8777 100644 --- a/src/main/java/pro/belbix/ethparser/web3/erc20/parser/TransferParser.java +++ b/src/main/java/pro/belbix/ethparser/web3/erc20/parser/TransferParser.java @@ -13,7 +13,6 @@ import pro.belbix.ethparser.model.tx.TokenTx; import pro.belbix.ethparser.properties.AppProperties; import pro.belbix.ethparser.properties.NetworkProperties; -import pro.belbix.ethparser.repositories.ErrorsRepository; import pro.belbix.ethparser.web3.EthBlockService; import pro.belbix.ethparser.web3.ParserInfo; import pro.belbix.ethparser.web3.Web3Functions; From ea6e9b5b287fac5509c749269f3914c5ab259556 Mon Sep 17 00:00:00 2001 From: alex Date: Mon, 28 Feb 2022 22:19:20 +0300 Subject: [PATCH 16/65] Change logic save transaction in db --- .../task/CovalenthqTransactionTask.java | 92 ++++++++----------- .../belbix/ethparser/web3/MethodDecoder.java | 1 + 2 files changed, 37 insertions(+), 56 deletions(-) diff --git a/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java b/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java index 669e4360..2f9eb001 100644 --- a/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java +++ b/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java @@ -10,7 +10,6 @@ import java.util.Collection; import java.util.HashSet; import java.util.LinkedList; -import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -67,7 +66,7 @@ public void start() { log.info("Begin parse vault tx"); var executor = Executors.newFixedThreadPool(20); - var futures = vaultRepository.findAll().stream() + vaultRepository.findAll().stream() .map(i -> CompletableFuture.runAsync(() -> getVaultTransaction(i), executor)) .collect(Collectors.toList()); @@ -89,56 +88,28 @@ private void getVaultTransaction(VaultEntity vault) { var currentBlock = web3Functions.fetchCurrentBlock(contract.getNetwork()); var endingBlock = startingBlock + MAX_BLOCK_RANGE; - var result = fetchTransactions(contract.getAddress(), contract.getNetwork(), startingBlock, endingBlock); while (currentBlock.longValue() >= endingBlock) { // TODO Maybe need to add one more block, because can find the same tx startingBlock = endingBlock; endingBlock += MAX_BLOCK_RANGE; - result.addAll( - fetchTransactions(contract.getAddress(), contract.getNetwork(), startingBlock, endingBlock)); + fetchTransactions(contract.getAddress(), contract.getNetwork(), startingBlock, endingBlock); } - var transactions = result.stream() - .map(CovalenthqTransactionHistoryItem::getLogs) - .flatMap(Collection::stream) - .filter(this::isDepositOrWithdraw) - .map(log -> toCovalenthqVaultTransaction(log, contract.getNetwork(), contract.getAddress())) - .filter(Objects::nonNull) - .collect(Collectors.toList()); - - var transactionWithoutDuplicate = new HashSet<>(transactions); - var transactionInDb = covalenthqVaultTransactionRepository.findAllByTransactionHashIn( - transactionWithoutDuplicate.stream() - .map(CovalenthqVaultTransaction::getTransactionHash) - .collect(Collectors.toList())); - transactions = transactionWithoutDuplicate.stream() - .filter(i1 -> - transactionInDb.stream().noneMatch(i2 -> - i1.getNetwork().equals(i2.getNetwork()) - && i1.getTransactionHash().equals(i2.getTransactionHash()) - && i1.getContractAddress().equals(i2.getContractAddress()) - && i1.getOwnerAddress().equals(i2.getOwnerAddress()) - ) - ) - .collect(Collectors.toList()); - - log.debug("Save list covalenthq tx: {}", transactionInDb); - covalenthqVaultTransactionRepository.saveAll(transactions); } catch (Exception e) { log.error("Get error getVaultTransaction", e); } } - private List fetchTransactions(String address, String network, long startingBlock, long endingBlock) { - return fetchTransactions(address, network, startingBlock, endingBlock, PAGINATION_SIZE, false); + private void fetchTransactions(String address, String network, long startingBlock, long endingBlock) { + fetchTransactions(address, network, startingBlock, endingBlock, PAGINATION_SIZE, false); } - private List fetchTransactions(String address, String network, long startingBlock, long endingBlock, int limit, boolean isAfterException) { + private void fetchTransactions(String address, String network, long startingBlock, long endingBlock, int limit, boolean isAfterException) { try { var page = 0; var transaction = covalenthqService.getTransactionByAddress(address, network, page, limit, startingBlock, endingBlock); - var result = new LinkedList(transaction.getData().getItems()); + var result = new LinkedList(transaction.getData().getItems()); var hasMore = transaction.getData().getPagination().isHasMore(); while (hasMore) { @@ -147,13 +118,40 @@ private List fetchTransactions(String address, hasMore = transaction.getData().getPagination().isHasMore(); result.addAll(transaction.getData().getItems()); } - return result; + + var transactions = result.stream() + .map(CovalenthqTransactionHistoryItem::getLogs) + .flatMap(Collection::stream) + .filter(this::isDepositOrWithdraw) + .map(log -> toCovalenthqVaultTransaction(log, network, address)) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + + var transactionWithoutDuplicate = new HashSet<>(transactions); + var transactionInDb = covalenthqVaultTransactionRepository.findAllByTransactionHashIn( + transactionWithoutDuplicate.stream() + .map(CovalenthqVaultTransaction::getTransactionHash) + .collect(Collectors.toList())); + transactions = transactionWithoutDuplicate.stream() + .filter(i1 -> + transactionInDb.stream().noneMatch(i2 -> + i1.getNetwork().equals(i2.getNetwork()) + && i1.getTransactionHash().equals(i2.getTransactionHash()) + && i1.getContractAddress().equals(i2.getContractAddress()) + && i1.getOwnerAddress().equals(i2.getOwnerAddress()) + ) + ) + .collect(Collectors.toList()); + + log.info("Save list covalenthq tx: {}", transactions); + covalenthqVaultTransactionRepository.saveAll(transactions); + } catch (IllegalStateException e) { - log.debug("Get a lot of data {} {} on block {} - {}", address, network, startingBlock, endingBlock); + log.error("Get a lot of data {} {} on block {} - {}", address, network, startingBlock, endingBlock); if (isAfterException) { throw e; } - return fetchTransactions(address, network, startingBlock, endingBlock, PAGINATION_SIZE_FOR_A_LOT_OF_DATA, true); + fetchTransactions(address, network, startingBlock, endingBlock, PAGINATION_SIZE_FOR_A_LOT_OF_DATA, true); } } @@ -200,24 +198,6 @@ private CovalenthqVaultTransactionType toCovalenthqVaultTransactionType(Covalent } private String getParamValue(CovalenthqTransactionHistoryItemLog item, String param) { - return getUniParamValue(item, param); - -// if (item == null || item.getDecoded() == null) { -// return StringUtils.EMPTY; -// } -// -// if (item.getDecoded().getParams() == null) { -// return getUniParamValue(item, param); -// } -// -// return item.getDecoded().getParams().stream() -// .filter(i -> param.equals(i.getName()) && i.getValue() instanceof String) -// .map(i -> (String)i.getValue()) -// .findFirst() -// .orElse(StringUtils.EMPTY); - } - - private String getUniParamValue(CovalenthqTransactionHistoryItemLog item, String param) { var ethLog = new Log(); ethLog.setTopics(item.getTopics()); ethLog.setData(item.getData()); diff --git a/src/main/java/pro/belbix/ethparser/web3/MethodDecoder.java b/src/main/java/pro/belbix/ethparser/web3/MethodDecoder.java index 0cff41a9..e495102f 100644 --- a/src/main/java/pro/belbix/ethparser/web3/MethodDecoder.java +++ b/src/main/java/pro/belbix/ethparser/web3/MethodDecoder.java @@ -92,6 +92,7 @@ public static List extractLogIndexedValues( ? i + 1 : i; + // TODO in some case get index out of String topic = topics.get(index); Type value = decodeIndexedValue(topic, indexedParameters.get(i)); indexedValues.add(value); From d667c0933a52f8ceea7c46b7f664c10cd16abc7e Mon Sep 17 00:00:00 2001 From: alex Date: Wed, 2 Mar 2022 00:17:56 +0300 Subject: [PATCH 17/65] Added sharePrice and tokenPrice --- scripts/sql/schema.sql | 13 ++++ .../profit/CovalenthqVaultTransaction.java | 3 + .../ethparser/entity/profit/SharePrice.java | 23 +++++++ .../ethparser/entity/profit/TokenPrice.java | 26 +++++++ .../belbix/ethparser/model/LogEventParam.java | 18 ----- .../repositories/SharePriceRepository.java | 8 +++ .../repositories/TokenPriceRepository.java | 8 +++ .../ethparser/service/SharePriceService.java | 46 +++++++++++++ .../ethparser/service/TokenPriceService.java | 69 +++++++++++++++++++ .../task/CovalenthqTransactionTask.java | 23 ++++++- .../belbix/ethparser/utils/ProfitUtils.java | 12 ++++ 11 files changed, 230 insertions(+), 19 deletions(-) create mode 100644 src/main/java/pro/belbix/ethparser/entity/profit/SharePrice.java create mode 100644 src/main/java/pro/belbix/ethparser/entity/profit/TokenPrice.java delete mode 100644 src/main/java/pro/belbix/ethparser/model/LogEventParam.java create mode 100644 src/main/java/pro/belbix/ethparser/repositories/SharePriceRepository.java create mode 100644 src/main/java/pro/belbix/ethparser/repositories/TokenPriceRepository.java create mode 100644 src/main/java/pro/belbix/ethparser/service/SharePriceService.java create mode 100644 src/main/java/pro/belbix/ethparser/service/TokenPriceService.java create mode 100644 src/main/java/pro/belbix/ethparser/utils/ProfitUtils.java diff --git a/scripts/sql/schema.sql b/scripts/sql/schema.sql index 99d3efde..f8b2c29c 100644 --- a/scripts/sql/schema.sql +++ b/scripts/sql/schema.sql @@ -1116,9 +1116,22 @@ create table covalenthq_vault_tx contract_decimal numeric(19, 0), contract_address varchar(255), owner_address varchar(255), + share_price numeric(60, 0), + token_price numeric(60, 6), value numeric(60, 0), signed_at timestamp, type varchar(255) ); +create table share_price +( + id varchar(255) primary key, + value numeric(60, 0) +); + +create table token_price +( + id varchar(255) primary key, + value numeric(60, 6) +); diff --git a/src/main/java/pro/belbix/ethparser/entity/profit/CovalenthqVaultTransaction.java b/src/main/java/pro/belbix/ethparser/entity/profit/CovalenthqVaultTransaction.java index 7e976505..016e8019 100644 --- a/src/main/java/pro/belbix/ethparser/entity/profit/CovalenthqVaultTransaction.java +++ b/src/main/java/pro/belbix/ethparser/entity/profit/CovalenthqVaultTransaction.java @@ -2,6 +2,7 @@ import java.math.BigDecimal; +import java.math.BigInteger; import java.time.LocalDateTime; import javax.persistence.Entity; import javax.persistence.GeneratedValue; @@ -28,6 +29,8 @@ public class CovalenthqVaultTransaction { String contractAddress; String ownerAddress; BigDecimal value; + BigInteger sharePrice; + Double tokenPrice; LocalDateTime signedAt; String type; } diff --git a/src/main/java/pro/belbix/ethparser/entity/profit/SharePrice.java b/src/main/java/pro/belbix/ethparser/entity/profit/SharePrice.java new file mode 100644 index 00000000..8a18b7a3 --- /dev/null +++ b/src/main/java/pro/belbix/ethparser/entity/profit/SharePrice.java @@ -0,0 +1,23 @@ +package pro.belbix.ethparser.entity.profit; + +import java.math.BigInteger; +import javax.persistence.Entity; +import javax.persistence.Id; +import javax.persistence.Table; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.FieldDefaults; + +@Entity +@Table(name = "share_price") +@Data +@FieldDefaults(level = AccessLevel.PRIVATE) +@NoArgsConstructor +@AllArgsConstructor +public class SharePrice { + @Id + String id; + BigInteger value; +} diff --git a/src/main/java/pro/belbix/ethparser/entity/profit/TokenPrice.java b/src/main/java/pro/belbix/ethparser/entity/profit/TokenPrice.java new file mode 100644 index 00000000..c18f7f3b --- /dev/null +++ b/src/main/java/pro/belbix/ethparser/entity/profit/TokenPrice.java @@ -0,0 +1,26 @@ +package pro.belbix.ethparser.entity.profit; + +import javax.persistence.Entity; +import javax.persistence.Id; +import javax.persistence.Table; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.FieldDefaults; + +@Entity +@Table(name = "token_price") +@Data +@FieldDefaults(level = AccessLevel.PRIVATE) +@NoArgsConstructor +@AllArgsConstructor +public class TokenPrice { + @Id + String id; + Double value; + + public static String toId(String vaultAddress, String block, String network) { + return String.join("_", vaultAddress, block, network); + } +} diff --git a/src/main/java/pro/belbix/ethparser/model/LogEventParam.java b/src/main/java/pro/belbix/ethparser/model/LogEventParam.java deleted file mode 100644 index 32c3cc1f..00000000 --- a/src/main/java/pro/belbix/ethparser/model/LogEventParam.java +++ /dev/null @@ -1,18 +0,0 @@ -package pro.belbix.ethparser.model; - -import lombok.AccessLevel; -import lombok.experimental.FieldDefaults; - -@FieldDefaults(makeFinal = true, level = AccessLevel.PUBLIC) -public enum LogEventParam { - WITHDRAW_TO("provider"), - WITHDRAW_VALUE("value"), - DEPOSIT_FROM("dst"), - DEPOSIT_VALUE("wad"); - - LogEventParam(String value) { - this.value = value; - } - - String value; -} diff --git a/src/main/java/pro/belbix/ethparser/repositories/SharePriceRepository.java b/src/main/java/pro/belbix/ethparser/repositories/SharePriceRepository.java new file mode 100644 index 00000000..e13fa1c0 --- /dev/null +++ b/src/main/java/pro/belbix/ethparser/repositories/SharePriceRepository.java @@ -0,0 +1,8 @@ +package pro.belbix.ethparser.repositories; + +import org.springframework.data.jpa.repository.JpaRepository; +import pro.belbix.ethparser.entity.profit.SharePrice; + +public interface SharePriceRepository extends JpaRepository { + +} diff --git a/src/main/java/pro/belbix/ethparser/repositories/TokenPriceRepository.java b/src/main/java/pro/belbix/ethparser/repositories/TokenPriceRepository.java new file mode 100644 index 00000000..59a7ecfc --- /dev/null +++ b/src/main/java/pro/belbix/ethparser/repositories/TokenPriceRepository.java @@ -0,0 +1,8 @@ +package pro.belbix.ethparser.repositories; + +import org.springframework.data.jpa.repository.JpaRepository; +import pro.belbix.ethparser.entity.profit.TokenPrice; + +public interface TokenPriceRepository extends JpaRepository { + +} diff --git a/src/main/java/pro/belbix/ethparser/service/SharePriceService.java b/src/main/java/pro/belbix/ethparser/service/SharePriceService.java new file mode 100644 index 00000000..f7950f7b --- /dev/null +++ b/src/main/java/pro/belbix/ethparser/service/SharePriceService.java @@ -0,0 +1,46 @@ +package pro.belbix.ethparser.service; + +import static pro.belbix.ethparser.web3.abi.FunctionsNames.GET_PRICE_PER_FULL_SHARE; + +import java.math.BigInteger; +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; +import lombok.experimental.FieldDefaults; +import lombok.extern.slf4j.Slf4j; +import org.springframework.cache.annotation.Cacheable; +import org.springframework.stereotype.Service; +import pro.belbix.ethparser.entity.profit.SharePrice; +import pro.belbix.ethparser.repositories.SharePriceRepository; +import pro.belbix.ethparser.utils.ProfitUtils; +import pro.belbix.ethparser.web3.abi.FunctionsUtils; + +@Service +@RequiredArgsConstructor +@Slf4j +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) +public class SharePriceService { + SharePriceRepository sharePriceRepository; + FunctionsUtils functionsUtils; + + @Cacheable("share_price") + public BigInteger getSharePrice(String vaultAddress, long block, String network) { + var id = ProfitUtils.toId(vaultAddress, String.valueOf(block), network); + var sharePrice = sharePriceRepository.findById(id); + + if (sharePrice.isPresent()) { + return sharePrice.get().getValue(); + } + + + var sharePriceInt = functionsUtils.callIntByName(GET_PRICE_PER_FULL_SHARE, vaultAddress, block, network) + .orElse(BigInteger.ZERO); + + if (sharePriceInt.compareTo(BigInteger.ZERO) == 0) { + log.info("Share price is zero {}", id); + } + sharePriceRepository.save(new SharePrice(id, sharePriceInt)); + + return sharePriceInt; + } + +} diff --git a/src/main/java/pro/belbix/ethparser/service/TokenPriceService.java b/src/main/java/pro/belbix/ethparser/service/TokenPriceService.java new file mode 100644 index 00000000..47b30212 --- /dev/null +++ b/src/main/java/pro/belbix/ethparser/service/TokenPriceService.java @@ -0,0 +1,69 @@ +package pro.belbix.ethparser.service; + +import static pro.belbix.ethparser.web3.abi.FunctionsNames.UNDERLYING; + +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; +import lombok.experimental.FieldDefaults; +import lombok.extern.slf4j.Slf4j; +import org.springframework.cache.annotation.Cacheable; +import org.springframework.stereotype.Service; +import pro.belbix.ethparser.entity.profit.TokenPrice; +import pro.belbix.ethparser.repositories.TokenPriceRepository; +import pro.belbix.ethparser.utils.ProfitUtils; +import pro.belbix.ethparser.web3.abi.FunctionService; +import pro.belbix.ethparser.web3.abi.FunctionsUtils; +import pro.belbix.ethparser.web3.contracts.ContractUtils; +import pro.belbix.ethparser.web3.prices.PriceProvider; + +@Service +@RequiredArgsConstructor +@Slf4j +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) +public class TokenPriceService { + private final static Double DEFAULT_RETURN_VALUE = 0.0; + + TokenPriceRepository tokenPriceRepository; + PriceProvider priceProvider; + FunctionsUtils functionsUtils; + FunctionService functionService; + + + @Cacheable("token_price") + public Double getTokenPrice(String vaultAddress, long block, String network) { + try { + + var id = ProfitUtils.toId(vaultAddress, String.valueOf(block), network); + var tokenPrice = tokenPriceRepository.findById(id); + + if (tokenPrice.isPresent()) { + return tokenPrice.get().getValue(); + } + + if (ContractUtils.isPsAddress(vaultAddress)) { + return priceProvider.getPriceForCoin(ContractUtils.getFarmAddress(network), block, network); + } + + var underlyingAddress = functionsUtils.callAddressByName(UNDERLYING, vaultAddress, block, network) + .orElseThrow(IllegalStateException::new); + + var price = priceProvider.getPriceForCoin(underlyingAddress, block, network); + +// if (functionService.isLp(underlyingAddress, block, network)) { +// log.error("Its LP func - {}", vaultAddress); +// } else { +// price = priceProvider.getPriceForCoin(underlyingAddress, block, network); +// } + + tokenPriceRepository.save(new TokenPrice(id, price)); + + return price; + } catch (IllegalStateException e) { + log.error("Can't fetch underlying token for {}", vaultAddress); + return DEFAULT_RETURN_VALUE; + } catch (Exception e) { + log.error("Can not get token price - {}", vaultAddress); + return DEFAULT_RETURN_VALUE; + } + } +} diff --git a/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java b/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java index 2f9eb001..a563f10f 100644 --- a/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java +++ b/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java @@ -33,6 +33,8 @@ import pro.belbix.ethparser.model.CovalenthqTransactionHistory.CovalenthqTransactionHistoryItems.CovalenthqTransactionHistoryItem.CovalenthqTransactionHistoryItemLog; import pro.belbix.ethparser.repositories.covalenthq.CovalenthqVaultTransactionRepository; import pro.belbix.ethparser.repositories.eth.VaultRepository; +import pro.belbix.ethparser.service.SharePriceService; +import pro.belbix.ethparser.service.TokenPriceService; import pro.belbix.ethparser.service.external.CovalenthqService; import pro.belbix.ethparser.web3.SimpleDecoder; import pro.belbix.ethparser.web3.Web3Functions; @@ -59,6 +61,9 @@ public class CovalenthqTransactionTask { CovalenthqService covalenthqService; CovalenthqVaultTransactionRepository covalenthqVaultTransactionRepository; Web3Functions web3Functions; + SharePriceService sharePriceService; + TokenPriceService tokenPriceService; + // everyday @Scheduled(fixedRate = 1000 * 60 * 60 * 24) @@ -141,9 +146,10 @@ private void fetchTransactions(String address, String network, long startingBloc && i1.getOwnerAddress().equals(i2.getOwnerAddress()) ) ) + .map(this::fillAdditionalParams) .collect(Collectors.toList()); - log.info("Save list covalenthq tx: {}", transactions); + log.info("Save list covalenthq tx count - {}", transactions.size()); covalenthqVaultTransactionRepository.saveAll(transactions); } catch (IllegalStateException e) { @@ -234,4 +240,19 @@ private String getParamValue(CovalenthqTransactionHistoryItemLog item, String pa ? (String) result.get(indexParam).getValue() : ((BigInteger) result.get(indexParam).getValue()).toString(); } + + private CovalenthqVaultTransaction fillAdditionalParams(CovalenthqVaultTransaction covalenthqVaultTransaction) { + var vaultAddress = covalenthqVaultTransaction.getContractAddress(); + var block = covalenthqVaultTransaction.getBlock(); + var network = covalenthqVaultTransaction.getNetwork(); + covalenthqVaultTransaction.setSharePrice( + sharePriceService.getSharePrice(vaultAddress, block, network) + ); + + covalenthqVaultTransaction.setTokenPrice( + tokenPriceService.getTokenPrice(vaultAddress, block, network) + ); + + return covalenthqVaultTransaction; + } } diff --git a/src/main/java/pro/belbix/ethparser/utils/ProfitUtils.java b/src/main/java/pro/belbix/ethparser/utils/ProfitUtils.java new file mode 100644 index 00000000..5f5cd647 --- /dev/null +++ b/src/main/java/pro/belbix/ethparser/utils/ProfitUtils.java @@ -0,0 +1,12 @@ +package pro.belbix.ethparser.utils; + +import lombok.experimental.UtilityClass; + +@UtilityClass +public class ProfitUtils { + private static final String ID_DELIMITER = "_"; + + public static String toId(String vaultAddress, String block, String network) { + return String.join(ID_DELIMITER, vaultAddress, block, network); + } +} From cdba053e7727f71a7602e5550ed9659d2ff99e5c Mon Sep 17 00:00:00 2001 From: alex Date: Thu, 3 Mar 2022 00:35:45 +0300 Subject: [PATCH 18/65] Calculate token price --- .../exceptions/CanNotFetchPriceException.java | 5 +++ .../model/CovalenthqHistoricalPrice.java | 19 +++++++++ .../repositories/eth/VaultRepository.java | 5 ++- .../ethparser/service/TokenPriceService.java | 42 +++++++++++++++---- .../service/external/CovalenthqService.java | 10 ++++- .../task/CovalenthqTransactionTask.java | 9 +++- .../service/task/HarvestPoolInfoTask.java | 3 +- .../service/task/HarvestVaultInfoTask.java | 3 +- .../pro/belbix/ethparser/utils/UrlUtils.java | 7 ++++ 9 files changed, 88 insertions(+), 15 deletions(-) create mode 100644 src/main/java/pro/belbix/ethparser/error/exceptions/CanNotFetchPriceException.java create mode 100644 src/main/java/pro/belbix/ethparser/model/CovalenthqHistoricalPrice.java diff --git a/src/main/java/pro/belbix/ethparser/error/exceptions/CanNotFetchPriceException.java b/src/main/java/pro/belbix/ethparser/error/exceptions/CanNotFetchPriceException.java new file mode 100644 index 00000000..a1e17615 --- /dev/null +++ b/src/main/java/pro/belbix/ethparser/error/exceptions/CanNotFetchPriceException.java @@ -0,0 +1,5 @@ +package pro.belbix.ethparser.error.exceptions; + +public class CanNotFetchPriceException extends RuntimeException{ + +} diff --git a/src/main/java/pro/belbix/ethparser/model/CovalenthqHistoricalPrice.java b/src/main/java/pro/belbix/ethparser/model/CovalenthqHistoricalPrice.java new file mode 100644 index 00000000..18a41619 --- /dev/null +++ b/src/main/java/pro/belbix/ethparser/model/CovalenthqHistoricalPrice.java @@ -0,0 +1,19 @@ +package pro.belbix.ethparser.model; + +import java.util.List; +import lombok.Data; + +@Data +public class CovalenthqHistoricalPrice { + private CovalenthqHistoricalPriceData data; + + @Data + public static class CovalenthqHistoricalPriceData { + private List prices; + + @Data + public static class CovalenthqHistoricalPriceDataPrice { + private Double price; + } + } +} diff --git a/src/main/java/pro/belbix/ethparser/repositories/eth/VaultRepository.java b/src/main/java/pro/belbix/ethparser/repositories/eth/VaultRepository.java index dd4f8ac2..8d386c57 100644 --- a/src/main/java/pro/belbix/ethparser/repositories/eth/VaultRepository.java +++ b/src/main/java/pro/belbix/ethparser/repositories/eth/VaultRepository.java @@ -4,7 +4,6 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; -import pro.belbix.ethparser.entity.contracts.ContractEntity; import pro.belbix.ethparser.entity.contracts.VaultEntity; public interface VaultRepository extends JpaRepository { @@ -29,4 +28,8 @@ VaultEntity findFirstByContract( + "left join fetch t.underlying f5 " + "where f1.network = :network") List fetchAllByNetwork(@Param("network") String network); + + @Query("select t from VaultEntity t " + + "where t.name not like 'fUni%'") + List findAllWithoutUni(); } diff --git a/src/main/java/pro/belbix/ethparser/service/TokenPriceService.java b/src/main/java/pro/belbix/ethparser/service/TokenPriceService.java index 47b30212..f35a3550 100644 --- a/src/main/java/pro/belbix/ethparser/service/TokenPriceService.java +++ b/src/main/java/pro/belbix/ethparser/service/TokenPriceService.java @@ -2,6 +2,9 @@ import static pro.belbix.ethparser.web3.abi.FunctionsNames.UNDERLYING; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.TimeZone; import lombok.AccessLevel; import lombok.RequiredArgsConstructor; import lombok.experimental.FieldDefaults; @@ -9,11 +12,15 @@ import org.springframework.cache.annotation.Cacheable; import org.springframework.stereotype.Service; import pro.belbix.ethparser.entity.profit.TokenPrice; +import pro.belbix.ethparser.error.exceptions.CanNotFetchPriceException; import pro.belbix.ethparser.repositories.TokenPriceRepository; +import pro.belbix.ethparser.service.external.CovalenthqService; import pro.belbix.ethparser.utils.ProfitUtils; +import pro.belbix.ethparser.web3.EthBlockService; import pro.belbix.ethparser.web3.abi.FunctionService; import pro.belbix.ethparser.web3.abi.FunctionsUtils; import pro.belbix.ethparser.web3.contracts.ContractUtils; +import pro.belbix.ethparser.web3.prices.PriceOracle; import pro.belbix.ethparser.web3.prices.PriceProvider; @Service @@ -27,6 +34,9 @@ public class TokenPriceService { PriceProvider priceProvider; FunctionsUtils functionsUtils; FunctionService functionService; + CovalenthqService covalenthqService; + EthBlockService ethBlockService; + PriceOracle priceOracle; @Cacheable("token_price") @@ -35,6 +45,7 @@ public Double getTokenPrice(String vaultAddress, long block, String network) { var id = ProfitUtils.toId(vaultAddress, String.valueOf(block), network); var tokenPrice = tokenPriceRepository.findById(id); + var price = DEFAULT_RETURN_VALUE; if (tokenPrice.isPresent()) { return tokenPrice.get().getValue(); @@ -47,23 +58,40 @@ public Double getTokenPrice(String vaultAddress, long block, String network) { var underlyingAddress = functionsUtils.callAddressByName(UNDERLYING, vaultAddress, block, network) .orElseThrow(IllegalStateException::new); - var price = priceProvider.getPriceForCoin(underlyingAddress, block, network); + if (PriceOracle.isAvailable(block, network)) { + price = priceOracle.getPriceForCoinOnChain(underlyingAddress, block, network); -// if (functionService.isLp(underlyingAddress, block, network)) { -// log.error("Its LP func - {}", vaultAddress); -// } else { -// price = priceProvider.getPriceForCoin(underlyingAddress, block, network); + tokenPriceRepository.save(new TokenPrice(id, price)); + return price; + } + +// var price = priceProvider.getPriceForCoin(underlyingAddress, block, network); + +// var blockDate = getDateByBlockNumber(block, network); +// var historicalPrice = covalenthqService.getPriceByContractAddress(underlyingAddress, blockDate, network); +// if (historicalPrice != null && historicalPrice.getData() != null +// && historicalPrice.getData().getPrices() != null +// && historicalPrice.getData().getPrices().size() > 0 +// && historicalPrice.getData().getPrices().get(0) != null +// ) { +// price = historicalPrice.getData().getPrices().get(0).getPrice(); // } tokenPriceRepository.save(new TokenPrice(id, price)); - return price; } catch (IllegalStateException e) { log.error("Can't fetch underlying token for {}", vaultAddress); return DEFAULT_RETURN_VALUE; } catch (Exception e) { log.error("Can not get token price - {}", vaultAddress); - return DEFAULT_RETURN_VALUE; + throw new CanNotFetchPriceException(); } } + + private String getDateByBlockNumber(long block, String network) { + long ts = ethBlockService.getTimestampSecForBlock(block, network); + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + dateFormat.setTimeZone(TimeZone.getTimeZone("GMT")); + return dateFormat.format(new Date(ts * 1000)); + } } diff --git a/src/main/java/pro/belbix/ethparser/service/external/CovalenthqService.java b/src/main/java/pro/belbix/ethparser/service/external/CovalenthqService.java index e8a42f92..21f4e180 100644 --- a/src/main/java/pro/belbix/ethparser/service/external/CovalenthqService.java +++ b/src/main/java/pro/belbix/ethparser/service/external/CovalenthqService.java @@ -15,6 +15,7 @@ import org.springframework.retry.annotation.Retryable; import org.springframework.stereotype.Service; import org.springframework.web.client.RestTemplate; +import pro.belbix.ethparser.model.CovalenthqHistoricalPrice; import pro.belbix.ethparser.model.CovalenthqTransactionHistory; import pro.belbix.ethparser.model.CovalenthqTransactionHistory.CovalenthqTransactionHistoryItems.CovalenthqTransactionHistoryItem; import pro.belbix.ethparser.model.CovalenthqTransactionHistory.CovalenthqTransactionHistoryItems.CovalenthqTransactionHistoryItem.CovalenthqTransactionHistoryItemLog; @@ -68,7 +69,6 @@ public long getCreatedBlockByLastTransaction(String address, String network) { } } - public CovalenthqTransactionHistory getTransactionByAddress(String address, String network, boolean isSortAsc, boolean isFullLogs, int page, int limit) { var url = String.format(CovalenthqUrl.TRANSACTION_HISTORY, @@ -98,6 +98,14 @@ public CovalenthqTransactionHistory getTransactionByAddress(String address, Stri } } + public CovalenthqHistoricalPrice getPriceByContractAddress(String contractAddress, String date, String network) { + var url = String.format(CovalenthqUrl.HISTORICAL_PRICE, + externalProperties.getCovalenthq().getUrl(), convertToNetwork(network), contractAddress, + externalProperties.getCovalenthq().getKey(), date, date); + + return restTemplate.getForObject(url, CovalenthqHistoricalPrice.class); + } + private String convertToNetwork(String network) { return Optional.ofNullable(CHAIN_BY_NETWORK.get(network)).orElse("1"); } diff --git a/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java b/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java index a563f10f..10f9caed 100644 --- a/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java +++ b/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java @@ -4,6 +4,7 @@ import static pro.belbix.ethparser.entity.profit.CovalenthqVaultTransactionType.DEPOSIT_UNI; import static pro.belbix.ethparser.entity.profit.CovalenthqVaultTransactionType.WITHDRAW; import static pro.belbix.ethparser.entity.profit.CovalenthqVaultTransactionType.WITHDRAW_UNI; +import static pro.belbix.ethparser.service.AbiProviderService.ETH_NETWORK; import java.math.BigDecimal; import java.math.BigInteger; @@ -29,6 +30,7 @@ import pro.belbix.ethparser.entity.contracts.VaultEntity; import pro.belbix.ethparser.entity.profit.CovalenthqVaultTransaction; import pro.belbix.ethparser.entity.profit.CovalenthqVaultTransactionType; +import pro.belbix.ethparser.error.exceptions.CanNotFetchPriceException; import pro.belbix.ethparser.model.CovalenthqTransactionHistory.CovalenthqTransactionHistoryItems.CovalenthqTransactionHistoryItem; import pro.belbix.ethparser.model.CovalenthqTransactionHistory.CovalenthqTransactionHistoryItems.CovalenthqTransactionHistoryItem.CovalenthqTransactionHistoryItemLog; import pro.belbix.ethparser.repositories.covalenthq.CovalenthqVaultTransactionRepository; @@ -69,9 +71,9 @@ public class CovalenthqTransactionTask { @Scheduled(fixedRate = 1000 * 60 * 60 * 24) public void start() { log.info("Begin parse vault tx"); - var executor = Executors.newFixedThreadPool(20); + var executor = Executors.newFixedThreadPool(2); - vaultRepository.findAll().stream() + vaultRepository.fetchAllByNetwork(ETH_NETWORK).stream() .map(i -> CompletableFuture.runAsync(() -> getVaultTransaction(i), executor)) .collect(Collectors.toList()); @@ -152,6 +154,9 @@ private void fetchTransactions(String address, String network, long startingBloc log.info("Save list covalenthq tx count - {}", transactions.size()); covalenthqVaultTransactionRepository.saveAll(transactions); + } catch (CanNotFetchPriceException e) { + log.error("Can not fetch price {} {}", address, network); + throw e; } catch (IllegalStateException e) { log.error("Get a lot of data {} {} on block {} - {}", address, network, startingBlock, endingBlock); if (isAfterException) { diff --git a/src/main/java/pro/belbix/ethparser/service/task/HarvestPoolInfoTask.java b/src/main/java/pro/belbix/ethparser/service/task/HarvestPoolInfoTask.java index 202e6b76..1337d237 100644 --- a/src/main/java/pro/belbix/ethparser/service/task/HarvestPoolInfoTask.java +++ b/src/main/java/pro/belbix/ethparser/service/task/HarvestPoolInfoTask.java @@ -13,7 +13,6 @@ import lombok.RequiredArgsConstructor; import lombok.experimental.FieldDefaults; import lombok.extern.slf4j.Slf4j; -import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import pro.belbix.ethparser.entity.contracts.ContractEntity; import pro.belbix.ethparser.entity.contracts.PoolEntity; @@ -45,7 +44,7 @@ public class HarvestPoolInfoTask { // each hour - @Scheduled(fixedRate = 1000 * 60 * 60) +// @Scheduled(fixedRate = 1000 * 60 * 60) public void start() { try { var response = harvestService.getPools(); diff --git a/src/main/java/pro/belbix/ethparser/service/task/HarvestVaultInfoTask.java b/src/main/java/pro/belbix/ethparser/service/task/HarvestVaultInfoTask.java index cf99d2f9..68f2737a 100644 --- a/src/main/java/pro/belbix/ethparser/service/task/HarvestVaultInfoTask.java +++ b/src/main/java/pro/belbix/ethparser/service/task/HarvestVaultInfoTask.java @@ -14,7 +14,6 @@ import lombok.RequiredArgsConstructor; import lombok.experimental.FieldDefaults; import lombok.extern.slf4j.Slf4j; -import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import pro.belbix.ethparser.entity.contracts.ContractEntity; import pro.belbix.ethparser.entity.contracts.VaultEntity; @@ -46,7 +45,7 @@ public class HarvestVaultInfoTask { // each hour - @Scheduled(fixedRate = 1000 * 60 * 60) +// @Scheduled(fixedRate = 1000 * 60 * 60) public void start() { try { var response = harvestService.getVaults(); diff --git a/src/main/java/pro/belbix/ethparser/utils/UrlUtils.java b/src/main/java/pro/belbix/ethparser/utils/UrlUtils.java index d9861804..f171968e 100644 --- a/src/main/java/pro/belbix/ethparser/utils/UrlUtils.java +++ b/src/main/java/pro/belbix/ethparser/utils/UrlUtils.java @@ -23,6 +23,13 @@ public interface CovalenthqUrl { + "starting-block=%s&" + "ending-block=%s&"; + String HISTORICAL_PRICE = "%spricing/historical_by_addresses_v2/%s/USD/%s/?" + + "quote-currency=USD&" + + "format=JSON&" + + "key=%s&" + + "from=%s&" + + "to=%s"; + String TRANSACTION_BY_CONTRACT_ID = "%s%s/address/%s/transfers_v2/?" + "contract-address=%s&" + "key=%s&" From bbd2bf84c25f6acc8122b7e1b5a1264f01e00819 Mon Sep 17 00:00:00 2001 From: alex Date: Fri, 4 Mar 2022 01:59:57 +0300 Subject: [PATCH 19/65] Find empty uniPairsToToken --- .../controllers/ProfitController.java | 7 ++++ .../entity/contracts/TokenEntity.java | 3 -- .../contracts/TokenToUniPairEntity.java | 3 -- .../CovalenthqVaultTransactionRepository.java | 1 + .../repositories/eth/ContractRepository.java | 5 +++ .../repositories/eth/TokenRepository.java | 4 ++- .../eth/TokenToUniPairRepository.java | 13 ++++++-- .../ethparser/service/ProfitService.java | 24 +++++++++----- .../ethparser/service/TokenPriceService.java | 12 +------ .../service/external/CovalenthqService.java | 1 + .../task/CovalenthqTransactionTask.java | 2 +- .../task/UpdateUniPairContractTask.java | 33 +++++++++++++++++++ .../web3/contracts/ContractLoader.java | 29 ++++++++++++++++ .../web3/contracts/db/ContractDbService.java | 2 +- .../ethparser/web3/prices/PriceProvider.java | 4 ++- 15 files changed, 111 insertions(+), 32 deletions(-) create mode 100644 src/main/java/pro/belbix/ethparser/service/task/UpdateUniPairContractTask.java diff --git a/src/main/java/pro/belbix/ethparser/controllers/ProfitController.java b/src/main/java/pro/belbix/ethparser/controllers/ProfitController.java index f412c2b8..41ad0d8f 100644 --- a/src/main/java/pro/belbix/ethparser/controllers/ProfitController.java +++ b/src/main/java/pro/belbix/ethparser/controllers/ProfitController.java @@ -3,7 +3,9 @@ import static pro.belbix.ethparser.service.AbiProviderService.ETH_NETWORK; import io.swagger.v3.oas.annotations.Parameter; +import java.math.BigDecimal; import lombok.extern.log4j.Log4j2; +import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RequestParam; @@ -45,4 +47,9 @@ public Double fetchProfitByVault( return profitService.calculationProfitByVaultForPeriod(address, network, start, end); } + + @GetMapping("api/profit") + public BigDecimal calculateProfit(@RequestParam String address, @RequestParam String network) { + return profitService.calculateProfit(address, network); + } } diff --git a/src/main/java/pro/belbix/ethparser/entity/contracts/TokenEntity.java b/src/main/java/pro/belbix/ethparser/entity/contracts/TokenEntity.java index e7cc8e97..b07a2db5 100644 --- a/src/main/java/pro/belbix/ethparser/entity/contracts/TokenEntity.java +++ b/src/main/java/pro/belbix/ethparser/entity/contracts/TokenEntity.java @@ -2,8 +2,6 @@ import javax.persistence.Entity; import javax.persistence.FetchType; -import javax.persistence.GeneratedValue; -import javax.persistence.GenerationType; import javax.persistence.Id; import javax.persistence.Index; import javax.persistence.JoinColumn; @@ -21,7 +19,6 @@ public class TokenEntity { @Id - @GeneratedValue(strategy = GenerationType.IDENTITY) private Integer id; @OneToOne(fetch = FetchType.EAGER) @JoinColumn(name = "contract", unique = true) diff --git a/src/main/java/pro/belbix/ethparser/entity/contracts/TokenToUniPairEntity.java b/src/main/java/pro/belbix/ethparser/entity/contracts/TokenToUniPairEntity.java index 42097f7c..c666cd51 100644 --- a/src/main/java/pro/belbix/ethparser/entity/contracts/TokenToUniPairEntity.java +++ b/src/main/java/pro/belbix/ethparser/entity/contracts/TokenToUniPairEntity.java @@ -2,8 +2,6 @@ import javax.persistence.Entity; import javax.persistence.FetchType; -import javax.persistence.GeneratedValue; -import javax.persistence.GenerationType; import javax.persistence.Id; import javax.persistence.JoinColumn; import javax.persistence.ManyToOne; @@ -18,7 +16,6 @@ public class TokenToUniPairEntity { @Id - @GeneratedValue(strategy = GenerationType.IDENTITY) private Integer id; private Long blockStart; diff --git a/src/main/java/pro/belbix/ethparser/repositories/covalenthq/CovalenthqVaultTransactionRepository.java b/src/main/java/pro/belbix/ethparser/repositories/covalenthq/CovalenthqVaultTransactionRepository.java index 6947f52a..8e7bd58d 100644 --- a/src/main/java/pro/belbix/ethparser/repositories/covalenthq/CovalenthqVaultTransactionRepository.java +++ b/src/main/java/pro/belbix/ethparser/repositories/covalenthq/CovalenthqVaultTransactionRepository.java @@ -8,4 +8,5 @@ public interface CovalenthqVaultTransactionRepository extends JpaRepository { List findAllByNetworkAndContractAddress(String network, String contractAddress, Pageable pageable); List findAllByTransactionHashIn(List transactionHashes); + List findAllByOwnerAddressAndNetwork(String ownerAddress, String network); } diff --git a/src/main/java/pro/belbix/ethparser/repositories/eth/ContractRepository.java b/src/main/java/pro/belbix/ethparser/repositories/eth/ContractRepository.java index 61dce004..142ab8a9 100644 --- a/src/main/java/pro/belbix/ethparser/repositories/eth/ContractRepository.java +++ b/src/main/java/pro/belbix/ethparser/repositories/eth/ContractRepository.java @@ -60,4 +60,9 @@ List findPoolsByVaultAddress( + "and c.type = :type") List findAllByNetworkAndInAddressAndType(String network, List addresses, int type); + @Query("select c from ContractEntity c " + + "left join UniPairEntity u on u.contract.id = c.id " + + "left join TokenToUniPairEntity t on t.uniPair.id = u.id " + + "where c.type = 2 and t.id is null") + List findAllUniPairContractWithoutData(); } diff --git a/src/main/java/pro/belbix/ethparser/repositories/eth/TokenRepository.java b/src/main/java/pro/belbix/ethparser/repositories/eth/TokenRepository.java index c420b50b..d0230abf 100644 --- a/src/main/java/pro/belbix/ethparser/repositories/eth/TokenRepository.java +++ b/src/main/java/pro/belbix/ethparser/repositories/eth/TokenRepository.java @@ -4,7 +4,6 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; -import pro.belbix.ethparser.entity.contracts.ContractEntity; import pro.belbix.ethparser.entity.contracts.TokenEntity; public interface TokenRepository extends JpaRepository { @@ -29,4 +28,7 @@ TokenEntity findFirstByName( + "left join fetch t.contract f1 " + "where f1.network = :network") List fetchAllByNetwork(@Param("network") String network); + + @Query("select max(t.id) from TokenEntity t") + int findMaxId(); } diff --git a/src/main/java/pro/belbix/ethparser/repositories/eth/TokenToUniPairRepository.java b/src/main/java/pro/belbix/ethparser/repositories/eth/TokenToUniPairRepository.java index 4db8eb7b..c3e0d77a 100644 --- a/src/main/java/pro/belbix/ethparser/repositories/eth/TokenToUniPairRepository.java +++ b/src/main/java/pro/belbix/ethparser/repositories/eth/TokenToUniPairRepository.java @@ -4,9 +4,7 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; -import pro.belbix.ethparser.entity.contracts.TokenEntity; import pro.belbix.ethparser.entity.contracts.TokenToUniPairEntity; -import pro.belbix.ethparser.entity.contracts.UniPairEntity; public interface TokenToUniPairRepository extends JpaRepository { @@ -58,4 +56,15 @@ List findByUniPair( @Param("network") String network ); + @Query("select t from TokenToUniPairEntity t " + + "join fetch t.uniPair f1 " + + "join fetch f1.contract c " + + "where lower(c.address) = :uniPairAdr and c.network = :network") + List findByUniPairAddress( + @Param("uniPairAdr") String uniPairAdr, + @Param("network") String network + ); + + @Query("select max(t.id) from TokenToUniPairEntity t") + int findMaxId(); } diff --git a/src/main/java/pro/belbix/ethparser/service/ProfitService.java b/src/main/java/pro/belbix/ethparser/service/ProfitService.java index 2c5d50ab..17e246a7 100644 --- a/src/main/java/pro/belbix/ethparser/service/ProfitService.java +++ b/src/main/java/pro/belbix/ethparser/service/ProfitService.java @@ -4,6 +4,7 @@ import static pro.belbix.ethparser.web3.abi.FunctionsNames.BALANCE_OF; import static pro.belbix.ethparser.web3.abi.FunctionsNames.GET_PRICE_PER_FULL_SHARE; +import java.math.BigDecimal; import java.math.BigInteger; import java.util.List; import java.util.Map; @@ -11,6 +12,8 @@ import lombok.extern.log4j.Log4j2; import org.springframework.stereotype.Service; import pro.belbix.ethparser.dto.v0.HarvestDTO; +import pro.belbix.ethparser.entity.profit.CovalenthqVaultTransaction; +import pro.belbix.ethparser.repositories.covalenthq.CovalenthqVaultTransactionRepository; import pro.belbix.ethparser.repositories.v0.HarvestRepository; import pro.belbix.ethparser.web3.EthBlockService; import pro.belbix.ethparser.web3.abi.FunctionsUtils; @@ -24,29 +27,32 @@ public class ProfitService { private final ContractDbService contractDbService; private final EthBlockService ethBlockService; private final FunctionsUtils functionsUtils; + private final CovalenthqVaultTransactionRepository covalenthqVaultTransactionRepository; public ProfitService(HarvestRepository harvestRepository, ContractDbService contractDbService, EthBlockService ethBlockService, - FunctionsUtils functionsUtils + FunctionsUtils functionsUtils, + CovalenthqVaultTransactionRepository covalenthqVaultTransactionRepository ) { this.harvestRepository = harvestRepository; this.contractDbService = contractDbService; this.ethBlockService = ethBlockService; this.functionsUtils = functionsUtils; - + this.covalenthqVaultTransactionRepository = covalenthqVaultTransactionRepository; } - public Long calculateProfit(String address) { - List tx = harvestRepository.findAllByOwner(address); - Long totalProfit = 0L; + public BigDecimal calculateProfit(String address, String network) { + var transactions = covalenthqVaultTransactionRepository.findAllByOwnerAddressAndNetwork(address, network); + BigDecimal totalProfit = BigDecimal.ZERO; - for (HarvestDTO i: tx) { - if (i.getMethodName().equals("Withdraw")) { - totalProfit += i.getUsdAmount(); + for (CovalenthqVaultTransaction i: transactions) { + var value = i.getValue().multiply(new BigDecimal(i.getSharePrice().toString())); + if (i.getType().equals("Withdraw")) { + totalProfit = totalProfit.add(value); } else { - totalProfit -= i.getUsdAmount(); + totalProfit = totalProfit.subtract(value); } } diff --git a/src/main/java/pro/belbix/ethparser/service/TokenPriceService.java b/src/main/java/pro/belbix/ethparser/service/TokenPriceService.java index f35a3550..5f167e28 100644 --- a/src/main/java/pro/belbix/ethparser/service/TokenPriceService.java +++ b/src/main/java/pro/belbix/ethparser/service/TokenPriceService.java @@ -65,17 +65,7 @@ public Double getTokenPrice(String vaultAddress, long block, String network) { return price; } -// var price = priceProvider.getPriceForCoin(underlyingAddress, block, network); - -// var blockDate = getDateByBlockNumber(block, network); -// var historicalPrice = covalenthqService.getPriceByContractAddress(underlyingAddress, blockDate, network); -// if (historicalPrice != null && historicalPrice.getData() != null -// && historicalPrice.getData().getPrices() != null -// && historicalPrice.getData().getPrices().size() > 0 -// && historicalPrice.getData().getPrices().get(0) != null -// ) { -// price = historicalPrice.getData().getPrices().get(0).getPrice(); -// } + price = priceProvider.getPriceForCoin(underlyingAddress, block, network); tokenPriceRepository.save(new TokenPrice(id, price)); return price; diff --git a/src/main/java/pro/belbix/ethparser/service/external/CovalenthqService.java b/src/main/java/pro/belbix/ethparser/service/external/CovalenthqService.java index 21f4e180..0521a210 100644 --- a/src/main/java/pro/belbix/ethparser/service/external/CovalenthqService.java +++ b/src/main/java/pro/belbix/ethparser/service/external/CovalenthqService.java @@ -83,6 +83,7 @@ public CovalenthqTransactionHistory getTransactionByAddress(String address, Stri } } + // if 507 response code try increase sleep time // if 524 response code try to resize page count @Retryable(value = Exception.class, maxAttempts = 4, backoff = @Backoff(delay = 1000)) public CovalenthqTransactionHistory getTransactionByAddress(String address, String network, diff --git a/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java b/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java index 10f9caed..cb17ec28 100644 --- a/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java +++ b/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java @@ -71,7 +71,7 @@ public class CovalenthqTransactionTask { @Scheduled(fixedRate = 1000 * 60 * 60 * 24) public void start() { log.info("Begin parse vault tx"); - var executor = Executors.newFixedThreadPool(2); + var executor = Executors.newFixedThreadPool(15); vaultRepository.fetchAllByNetwork(ETH_NETWORK).stream() .map(i -> CompletableFuture.runAsync(() -> getVaultTransaction(i), executor)) diff --git a/src/main/java/pro/belbix/ethparser/service/task/UpdateUniPairContractTask.java b/src/main/java/pro/belbix/ethparser/service/task/UpdateUniPairContractTask.java new file mode 100644 index 00000000..8dd745a2 --- /dev/null +++ b/src/main/java/pro/belbix/ethparser/service/task/UpdateUniPairContractTask.java @@ -0,0 +1,33 @@ +package pro.belbix.ethparser.service.task; + +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; +import lombok.experimental.FieldDefaults; +import lombok.extern.slf4j.Slf4j; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import pro.belbix.ethparser.entity.contracts.ContractEntity; +import pro.belbix.ethparser.repositories.eth.ContractRepository; +import pro.belbix.ethparser.web3.contracts.ContractLoader; + +@Service +@RequiredArgsConstructor +@Slf4j +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) +public class UpdateUniPairContractTask { + ContractRepository contractRepository; + ContractLoader contractLoader; + + @Scheduled(fixedRate = 1000 * 60 * 60 * 24) + public void start() { + contractRepository.findAllUniPairContractWithoutData().stream() + .forEach(this::fetchUniPairContact); + } + + public void fetchUniPairContact(ContractEntity contract) { + log.info("Begin fetch uniPairsToToken for {} {}", contract.getAddress(), contract.getNetwork()); + var token = contractLoader.loadToken(contract, contract.getNetwork(), contract.getCreated()); + var uniPairsToToken = contractLoader.linkUniPairsToToken(contract.getAddress(), contract.getCreated(), token, contract.getNetwork()); + log.info("Finish fetch uniPairsToToken - {}", uniPairsToToken); + } +} diff --git a/src/main/java/pro/belbix/ethparser/web3/contracts/ContractLoader.java b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractLoader.java index 6046ee20..e546bdd6 100644 --- a/src/main/java/pro/belbix/ethparser/web3/contracts/ContractLoader.java +++ b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractLoader.java @@ -109,11 +109,16 @@ public ContractEntity load(PureEthContractInfo contractInfo) { public void loadToken(TokenContract contract, String network, long block) { ContractEntity tokenContract = load(contract); + loadToken(tokenContract, network, block); + } + public TokenEntity loadToken(ContractEntity tokenContract, String network, long block) { TokenEntity tokenEntity = tokenRepository .findFirstByAddress(tokenContract.getAddress(), network); if (tokenEntity == null) { tokenEntity = new TokenEntity(); + var maxId = tokenRepository.findMaxId() + 1; + tokenEntity.setId(maxId); tokenEntity.setContract(tokenContract); enrichToken(tokenEntity, block, network); tokenRepository.save(tokenEntity); @@ -121,6 +126,8 @@ public void loadToken(TokenContract contract, String network, long block) { enrichToken(tokenEntity, block, network); tokenRepository.save(tokenEntity); } + + return tokenEntity; } public void loadVault(SimpleContract vault, String network, long block) { @@ -432,6 +439,26 @@ public void linkUniPairsToToken(TokenContract tokenContract, String network) { } } + public TokenToUniPairEntity linkUniPairsToToken(String address, long block, TokenEntity tokenEntity, String network) { + UniPairEntity uniPair; + if (address.startsWith("0x")) { + uniPair = uniPairRepository.findFirstByAddress(address, network); + } else { + uniPair = uniPairRepository.findFirstByName(address, network); + } + + if (uniPair == null) { + log.error("Not found lp for {} on {}", address, network); + return null; + } + + if (tokenEntity == null) { + log.error("Token is null for " + address); + return null; + } + return findOrCreateTokenToUniPair(tokenEntity, uniPair, block, network); + } + private TokenToUniPairEntity findOrCreateTokenToUniPair( TokenEntity token, UniPairEntity uniPair, @@ -456,7 +483,9 @@ private TokenToUniPairEntity findOrCreateTokenToUniPair( if (pairByLp != null && !pairByLp.isEmpty()) { log.info("We already had linked " + uniPair.getContract().getName()); } + var id = tokenToUniPairRepository.findMaxId() + 1; tokenToUniPairEntity = new TokenToUniPairEntity(); + tokenToUniPairEntity.setId(id); tokenToUniPairEntity.setToken(token); tokenToUniPairEntity.setUniPair(uniPair); tokenToUniPairEntity.setBlockStart(blockStart); diff --git a/src/main/java/pro/belbix/ethparser/web3/contracts/db/ContractDbService.java b/src/main/java/pro/belbix/ethparser/web3/contracts/db/ContractDbService.java index 9f5fa6ff..a6bd05ef 100644 --- a/src/main/java/pro/belbix/ethparser/web3/contracts/db/ContractDbService.java +++ b/src/main/java/pro/belbix/ethparser/web3/contracts/db/ContractDbService.java @@ -158,7 +158,7 @@ public Optional findLpByAddress(String address, String network) { public Optional findPairByToken(String tokenAddress, long block, String network) { List pairs = tokenToUniPairRepository - .findByToken(tokenAddress.toLowerCase(), network); + .findByUniPairAddress(tokenAddress.toLowerCase(), network); if (pairs == null || pairs.isEmpty()) { return Optional.empty(); } diff --git a/src/main/java/pro/belbix/ethparser/web3/prices/PriceProvider.java b/src/main/java/pro/belbix/ethparser/web3/prices/PriceProvider.java index a24356f4..190870cd 100644 --- a/src/main/java/pro/belbix/ethparser/web3/prices/PriceProvider.java +++ b/src/main/java/pro/belbix/ethparser/web3/prices/PriceProvider.java @@ -15,6 +15,7 @@ import org.springframework.stereotype.Service; import org.web3j.tuples.generated.Tuple2; import pro.belbix.ethparser.entity.contracts.ContractEntity; +import pro.belbix.ethparser.error.exceptions.CanNotFetchPriceException; import pro.belbix.ethparser.properties.AppProperties; import pro.belbix.ethparser.repositories.v0.PriceRepository; import pro.belbix.ethparser.web3.abi.FunctionsUtils; @@ -204,7 +205,8 @@ private double getPriceForCoinFromEthLegacy( } catch (Exception ignore) { } log.error("Not found lp for {}, block: {}, network: {}", address, block, network); - return 0; + throw new CanNotFetchPriceException(); +// return 0; } Tuple2 reserves = functionsUtils From 1de9c7b9df30d4fc0f34fd029992d19b999949c4 Mon Sep 17 00:00:00 2001 From: alex Date: Sun, 6 Mar 2022 02:11:36 +0300 Subject: [PATCH 20/65] Resolve problem with price --- .../exceptions/CanNotFetchPriceException.java | 11 +++ .../model/CovalenthqTransactionHistory.java | 9 +++ .../ethparser/service/TokenPriceService.java | 42 ++++------ .../service/external/CovalenthqService.java | 77 +++++++++---------- .../task/CovalenthqTransactionTask.java | 4 +- .../pro/belbix/ethparser/utils/UrlUtils.java | 49 ++++-------- .../web3/contracts/ContractConstants.java | 1 + .../web3/contracts/db/ContractDbService.java | 2 +- .../ethparser/web3/prices/PriceProvider.java | 4 +- 9 files changed, 91 insertions(+), 108 deletions(-) diff --git a/src/main/java/pro/belbix/ethparser/error/exceptions/CanNotFetchPriceException.java b/src/main/java/pro/belbix/ethparser/error/exceptions/CanNotFetchPriceException.java index a1e17615..7fa53e56 100644 --- a/src/main/java/pro/belbix/ethparser/error/exceptions/CanNotFetchPriceException.java +++ b/src/main/java/pro/belbix/ethparser/error/exceptions/CanNotFetchPriceException.java @@ -1,5 +1,16 @@ package pro.belbix.ethparser.error.exceptions; +import lombok.AccessLevel; +import lombok.Data; +import lombok.experimental.FieldDefaults; + +@Data +@FieldDefaults(level = AccessLevel.PRIVATE) public class CanNotFetchPriceException extends RuntimeException{ + String message; + public CanNotFetchPriceException() {} + public CanNotFetchPriceException(String message) { + this.message = message; + } } diff --git a/src/main/java/pro/belbix/ethparser/model/CovalenthqTransactionHistory.java b/src/main/java/pro/belbix/ethparser/model/CovalenthqTransactionHistory.java index 9ff619e5..40a998d7 100644 --- a/src/main/java/pro/belbix/ethparser/model/CovalenthqTransactionHistory.java +++ b/src/main/java/pro/belbix/ethparser/model/CovalenthqTransactionHistory.java @@ -1,6 +1,11 @@ package pro.belbix.ethparser.model; +import com.fasterxml.jackson.annotation.JsonFormat; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.fasterxml.jackson.datatype.jsr310.deser.LocalDateTimeDeserializer; +import com.fasterxml.jackson.datatype.jsr310.ser.LocalDateTimeSerializer; import java.time.LocalDateTime; import java.util.List; import lombok.Data; @@ -17,6 +22,10 @@ public static class CovalenthqTransactionHistoryItems { @Data public static class CovalenthqTransactionHistoryItem { @JsonProperty("block_signed_at") + @JsonSerialize(using = LocalDateTimeSerializer.class) + @JsonDeserialize(using = LocalDateTimeDeserializer.class) + @JsonFormat(pattern = "yyyy-MM-dd'T'HH:mm:ss'Z'") + // 2021-01-03T16:55:11Z private LocalDateTime signedAt; @JsonProperty("block_height") private long blockHeight; diff --git a/src/main/java/pro/belbix/ethparser/service/TokenPriceService.java b/src/main/java/pro/belbix/ethparser/service/TokenPriceService.java index 5f167e28..95eba1e1 100644 --- a/src/main/java/pro/belbix/ethparser/service/TokenPriceService.java +++ b/src/main/java/pro/belbix/ethparser/service/TokenPriceService.java @@ -2,9 +2,7 @@ import static pro.belbix.ethparser.web3.abi.FunctionsNames.UNDERLYING; -import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.TimeZone; +import java.math.BigDecimal; import lombok.AccessLevel; import lombok.RequiredArgsConstructor; import lombok.experimental.FieldDefaults; @@ -14,13 +12,9 @@ import pro.belbix.ethparser.entity.profit.TokenPrice; import pro.belbix.ethparser.error.exceptions.CanNotFetchPriceException; import pro.belbix.ethparser.repositories.TokenPriceRepository; -import pro.belbix.ethparser.service.external.CovalenthqService; import pro.belbix.ethparser.utils.ProfitUtils; -import pro.belbix.ethparser.web3.EthBlockService; -import pro.belbix.ethparser.web3.abi.FunctionService; import pro.belbix.ethparser.web3.abi.FunctionsUtils; import pro.belbix.ethparser.web3.contracts.ContractUtils; -import pro.belbix.ethparser.web3.prices.PriceOracle; import pro.belbix.ethparser.web3.prices.PriceProvider; @Service @@ -33,14 +27,10 @@ public class TokenPriceService { TokenPriceRepository tokenPriceRepository; PriceProvider priceProvider; FunctionsUtils functionsUtils; - FunctionService functionService; - CovalenthqService covalenthqService; - EthBlockService ethBlockService; - PriceOracle priceOracle; @Cacheable("token_price") - public Double getTokenPrice(String vaultAddress, long block, String network) { + public Double getTokenPrice(String vaultAddress, BigDecimal amount, long block, String network) { try { var id = ProfitUtils.toId(vaultAddress, String.valueOf(block), network); @@ -56,32 +46,26 @@ public Double getTokenPrice(String vaultAddress, long block, String network) { } var underlyingAddress = functionsUtils.callAddressByName(UNDERLYING, vaultAddress, block, network) - .orElseThrow(IllegalStateException::new); + .orElseThrow(() -> { + log.error("Can not fetch underlying address for {} {}", vaultAddress, network); + throw new IllegalStateException(); + }); - if (PriceOracle.isAvailable(block, network)) { - price = priceOracle.getPriceForCoinOnChain(underlyingAddress, block, network); - - tokenPriceRepository.save(new TokenPrice(id, price)); - return price; + if (isToken(underlyingAddress, block, network)) { + price = priceProvider.getPriceForCoin(underlyingAddress, block, network); + } else { + price = priceProvider.getLpTokenUsdPrice(underlyingAddress, amount.doubleValue(), block, network); } - price = priceProvider.getPriceForCoin(underlyingAddress, block, network); - tokenPriceRepository.save(new TokenPrice(id, price)); return price; - } catch (IllegalStateException e) { - log.error("Can't fetch underlying token for {}", vaultAddress); - return DEFAULT_RETURN_VALUE; } catch (Exception e) { - log.error("Can not get token price - {}", vaultAddress); + log.error("Can not get token price - {}", vaultAddress, e); throw new CanNotFetchPriceException(); } } - private String getDateByBlockNumber(long block, String network) { - long ts = ethBlockService.getTimestampSecForBlock(block, network); - SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); - dateFormat.setTimeZone(TimeZone.getTimeZone("GMT")); - return dateFormat.format(new Date(ts * 1000)); + private boolean isToken(String underlyingAddress, long block, String network) { + return functionsUtils.callReserves(underlyingAddress, block, network) == null; } } diff --git a/src/main/java/pro/belbix/ethparser/service/external/CovalenthqService.java b/src/main/java/pro/belbix/ethparser/service/external/CovalenthqService.java index 0521a210..79b94375 100644 --- a/src/main/java/pro/belbix/ethparser/service/external/CovalenthqService.java +++ b/src/main/java/pro/belbix/ethparser/service/external/CovalenthqService.java @@ -4,23 +4,22 @@ import static pro.belbix.ethparser.service.AbiProviderService.ETH_NETWORK; import static pro.belbix.ethparser.service.AbiProviderService.MATIC_NETWORK; -import java.util.List; import java.util.Map; import java.util.Optional; import lombok.AccessLevel; import lombok.RequiredArgsConstructor; import lombok.experimental.FieldDefaults; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; import org.springframework.retry.annotation.Backoff; import org.springframework.retry.annotation.Retryable; import org.springframework.stereotype.Service; import org.springframework.web.client.RestTemplate; -import pro.belbix.ethparser.model.CovalenthqHistoricalPrice; import pro.belbix.ethparser.model.CovalenthqTransactionHistory; import pro.belbix.ethparser.model.CovalenthqTransactionHistory.CovalenthqTransactionHistoryItems.CovalenthqTransactionHistoryItem; -import pro.belbix.ethparser.model.CovalenthqTransactionHistory.CovalenthqTransactionHistoryItems.CovalenthqTransactionHistoryItem.CovalenthqTransactionHistoryItemLog; import pro.belbix.ethparser.properties.ExternalProperties; -import pro.belbix.ethparser.utils.UrlUtils.CovalenthqUrl; +import pro.belbix.ethparser.utils.UrlUtils.Covalenthq.Params; +import pro.belbix.ethparser.utils.UrlUtils.Covalenthq.Url; @Service @RequiredArgsConstructor @@ -28,8 +27,8 @@ @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) public class CovalenthqService { + private final static String DEFAULT_CHAIN = "1"; private final static int TRANSFER_LIMIT = 3; - private final static String TRANSFER_LOG_NAME = "Transfer"; private final static Map CHAIN_BY_NETWORK = Map.of( MATIC_NETWORK, "137", ETH_NETWORK, "1", @@ -71,26 +70,43 @@ public long getCreatedBlockByLastTransaction(String address, String network) { public CovalenthqTransactionHistory getTransactionByAddress(String address, String network, boolean isSortAsc, boolean isFullLogs, int page, int limit) { - var url = String.format(CovalenthqUrl.TRANSACTION_HISTORY, - externalProperties.getCovalenthq().getUrl(), convertToNetwork(network), address, isSortAsc, - isFullLogs, - externalProperties.getCovalenthq().getKey(), page, limit); - try { - return restTemplate.getForObject(url, CovalenthqTransactionHistory.class); - } catch (Exception e) { - log.error("Error during call {}", url, e); - throw new IllegalStateException(e); - } + var params = String.join(StringUtils.EMPTY, + Params.QUOTE_CURRENCY, + Params.FORMAT, + String.format(Params.BLOCK_SIGNED_AT_ASC, isSortAsc), + String.format(Params.NO_LOGS, isFullLogs), + String.format(Params.KEY, externalProperties.getCovalenthq().getKey()), + String.format(Params.PAGE_NUMBER, page), + String.format(Params.PAGE_SIZE, limit) + ); + + var url = String.format(Url.TRANSACTION_HISTORY, externalProperties.getCovalenthq().getUrl(), convertToNetwork(network), address, params); + + return getTransactionByAddress(url); + } + + public CovalenthqTransactionHistory getTransactionByAddress(String address, String network, + int page, int limit, long startingBlock, long endingBlock) { + var params = String.join(StringUtils.EMPTY, + Params.QUOTE_CURRENCY, + Params.FORMAT, + String.format(Params.BLOCK_SIGNED_AT_ASC, true), + String.format(Params.NO_LOGS, false), + String.format(Params.KEY, externalProperties.getCovalenthq().getKey()), + String.format(Params.PAGE_NUMBER, page), + String.format(Params.PAGE_SIZE, limit), + String.format(Params.START_BLOCK, startingBlock), + String.format(Params.END_BLOCK, endingBlock) + ); + + var url = String.format(Url.TRANSACTION_HISTORY, externalProperties.getCovalenthq().getUrl(), convertToNetwork(network), address, params); + return getTransactionByAddress(url); } // if 507 response code try increase sleep time // if 524 response code try to resize page count @Retryable(value = Exception.class, maxAttempts = 4, backoff = @Backoff(delay = 1000)) - public CovalenthqTransactionHistory getTransactionByAddress(String address, String network, - int page, int limit, long startingBlock, long endingBlock) { - var url = String.format(CovalenthqUrl.TRANSACTION_HISTORY_WITH_BLOCK_RANGE, - externalProperties.getCovalenthq().getUrl(), convertToNetwork(network), address, - externalProperties.getCovalenthq().getKey(), page, limit, startingBlock, endingBlock); + public CovalenthqTransactionHistory getTransactionByAddress(String url) { try { return restTemplate.getForObject(url, CovalenthqTransactionHistory.class); } catch (Exception e) { @@ -99,33 +115,16 @@ public CovalenthqTransactionHistory getTransactionByAddress(String address, Stri } } - public CovalenthqHistoricalPrice getPriceByContractAddress(String contractAddress, String date, String network) { - var url = String.format(CovalenthqUrl.HISTORICAL_PRICE, - externalProperties.getCovalenthq().getUrl(), convertToNetwork(network), contractAddress, - externalProperties.getCovalenthq().getKey(), date, date); - - return restTemplate.getForObject(url, CovalenthqHistoricalPrice.class); - } - private String convertToNetwork(String network) { - return Optional.ofNullable(CHAIN_BY_NETWORK.get(network)).orElse("1"); + return Optional.ofNullable(CHAIN_BY_NETWORK.get(network)).orElse(DEFAULT_CHAIN); } private CovalenthqTransactionHistoryItem findCovalenthqTransactionItem( CovalenthqTransactionHistory result) { return result.getData().getItems().stream() - .filter(items -> items.getLogs() != null && notHasTransferInLog(items.getLogs())) + .filter(items -> items.getLogs() != null) .findFirst() .orElse(null); } - - private boolean notHasTransferInLog(List logs) { -// for (CovalenthqTransactionHistoryItemLog log : logs) { -// if (log.getDecoded() != null && TRANSFER_LOG_NAME.equals(log.getDecoded().getName())) { -// return false; -// } -// } - return true; - } } diff --git a/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java b/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java index cb17ec28..20a91d0e 100644 --- a/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java +++ b/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java @@ -71,7 +71,7 @@ public class CovalenthqTransactionTask { @Scheduled(fixedRate = 1000 * 60 * 60 * 24) public void start() { log.info("Begin parse vault tx"); - var executor = Executors.newFixedThreadPool(15); + var executor = Executors.newFixedThreadPool(10); vaultRepository.fetchAllByNetwork(ETH_NETWORK).stream() .map(i -> CompletableFuture.runAsync(() -> getVaultTransaction(i), executor)) @@ -255,7 +255,7 @@ private CovalenthqVaultTransaction fillAdditionalParams(CovalenthqVaultTransacti ); covalenthqVaultTransaction.setTokenPrice( - tokenPriceService.getTokenPrice(vaultAddress, block, network) + tokenPriceService.getTokenPrice(vaultAddress, covalenthqVaultTransaction.getValue(), block, network) ); return covalenthqVaultTransaction; diff --git a/src/main/java/pro/belbix/ethparser/utils/UrlUtils.java b/src/main/java/pro/belbix/ethparser/utils/UrlUtils.java index f171968e..4858f194 100644 --- a/src/main/java/pro/belbix/ethparser/utils/UrlUtils.java +++ b/src/main/java/pro/belbix/ethparser/utils/UrlUtils.java @@ -2,41 +2,22 @@ public class UrlUtils { - public interface CovalenthqUrl { - String TRANSACTION_HISTORY = "%s%s/address/%s/transactions_v2/?" - + "quote-currency=USD&" - + "format=JSON&" - + "block-signed-at-asc=%s&" - + "no-logs=%s&" - + "key=%s&" - + "page-number=%s&" - + "page-size=%s&"; - // TODO move params to restTemplate - String TRANSACTION_HISTORY_WITH_BLOCK_RANGE = "%s%s/address/%s/transactions_v2/?" - + "quote-currency=USD&" - + "format=JSON&" - + "block-signed-at-asc=true&" - + "no-logs=false&" - + "key=%s&" - + "page-number=%s&" - + "page-size=%s&" - + "starting-block=%s&" - + "ending-block=%s&"; + public interface Covalenthq { + interface Url { + String TRANSACTION_HISTORY = "%s%s/address/%s/transactions_v2/?%s"; + } + interface Params { + String QUOTE_CURRENCY = "quote-currency=USD&"; + String FORMAT = "format=JSON&"; + String BLOCK_SIGNED_AT_ASC = "block-signed-at-asc=%s&"; + String NO_LOGS = "no-logs=%s&"; + String KEY = "key=%s&"; + String PAGE_NUMBER = "page-number=%s&"; + String PAGE_SIZE = "page-size=%s&"; + String START_BLOCK = "starting-block=%s&"; + String END_BLOCK = "ending-block=%s&"; + } - String HISTORICAL_PRICE = "%spricing/historical_by_addresses_v2/%s/USD/%s/?" - + "quote-currency=USD&" - + "format=JSON&" - + "key=%s&" - + "from=%s&" - + "to=%s"; - - String TRANSACTION_BY_CONTRACT_ID = "%s%s/address/%s/transfers_v2/?" - + "contract-address=%s&" - + "key=%s&" - + "page-number=%s&" - + "page-size=%s&" - + "starting-block=%s&" - + "ending-block=%s"; } public interface HarvestUrl { diff --git a/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstants.java b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstants.java index 35e66e7f..bee7d589 100644 --- a/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstants.java +++ b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstants.java @@ -155,6 +155,7 @@ public class ContractConstants { "0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48".toLowerCase(), //USDC "0xe9e7cea3dedca5984780bafc599bd69add087d56".toLowerCase(), //BUSD "0xdAC17F958D2ee523a2206206994597C13D831ec7".toLowerCase(), //USDT + "0x0000000000085d4780B73119b644AE5ecd22b376".toLowerCase(), //TUSD "0x2791bca1f2de4661ed88a30c99a7a9449aa84174".toLowerCase(), //matic USDC "0xc2132d05d31c914a87c6611c10748aeb04b58e8f".toLowerCase() //matic USDT ); diff --git a/src/main/java/pro/belbix/ethparser/web3/contracts/db/ContractDbService.java b/src/main/java/pro/belbix/ethparser/web3/contracts/db/ContractDbService.java index a6bd05ef..9f5fa6ff 100644 --- a/src/main/java/pro/belbix/ethparser/web3/contracts/db/ContractDbService.java +++ b/src/main/java/pro/belbix/ethparser/web3/contracts/db/ContractDbService.java @@ -158,7 +158,7 @@ public Optional findLpByAddress(String address, String network) { public Optional findPairByToken(String tokenAddress, long block, String network) { List pairs = tokenToUniPairRepository - .findByUniPairAddress(tokenAddress.toLowerCase(), network); + .findByToken(tokenAddress.toLowerCase(), network); if (pairs == null || pairs.isEmpty()) { return Optional.empty(); } diff --git a/src/main/java/pro/belbix/ethparser/web3/prices/PriceProvider.java b/src/main/java/pro/belbix/ethparser/web3/prices/PriceProvider.java index 190870cd..a24356f4 100644 --- a/src/main/java/pro/belbix/ethparser/web3/prices/PriceProvider.java +++ b/src/main/java/pro/belbix/ethparser/web3/prices/PriceProvider.java @@ -15,7 +15,6 @@ import org.springframework.stereotype.Service; import org.web3j.tuples.generated.Tuple2; import pro.belbix.ethparser.entity.contracts.ContractEntity; -import pro.belbix.ethparser.error.exceptions.CanNotFetchPriceException; import pro.belbix.ethparser.properties.AppProperties; import pro.belbix.ethparser.repositories.v0.PriceRepository; import pro.belbix.ethparser.web3.abi.FunctionsUtils; @@ -205,8 +204,7 @@ private double getPriceForCoinFromEthLegacy( } catch (Exception ignore) { } log.error("Not found lp for {}, block: {}, network: {}", address, block, network); - throw new CanNotFetchPriceException(); -// return 0; + return 0; } Tuple2 reserves = functionsUtils From 86535ba7a1596d021e95ce4dcba3e24690441215 Mon Sep 17 00:00:00 2001 From: alex Date: Mon, 7 Mar 2022 02:17:16 +0300 Subject: [PATCH 21/65] Calculate price --- .../controllers/ProfitController.java | 6 +++--- .../ethparser/entity/profit/TokenPrice.java | 4 ---- .../model/CovalenthqTransactionHistory.java | 11 +++++------ .../belbix/ethparser/model/ProfitResult.java | 11 +++++++++++ .../ethparser/service/ProfitService.java | 18 +++++++++++++++--- .../ethparser/service/SharePriceService.java | 2 ++ .../ethparser/service/TokenPriceService.java | 3 ++- .../task/CovalenthqTransactionTask.java | 4 ++-- .../web3/contracts/ContractConstants.java | 15 +++++++++++++++ .../web3/contracts/ContractUtils.java | 8 ++++++++ 10 files changed, 63 insertions(+), 19 deletions(-) create mode 100644 src/main/java/pro/belbix/ethparser/model/ProfitResult.java diff --git a/src/main/java/pro/belbix/ethparser/controllers/ProfitController.java b/src/main/java/pro/belbix/ethparser/controllers/ProfitController.java index 41ad0d8f..095b8263 100644 --- a/src/main/java/pro/belbix/ethparser/controllers/ProfitController.java +++ b/src/main/java/pro/belbix/ethparser/controllers/ProfitController.java @@ -3,13 +3,13 @@ import static pro.belbix.ethparser.service.AbiProviderService.ETH_NETWORK; import io.swagger.v3.oas.annotations.Parameter; -import java.math.BigDecimal; import lombok.extern.log4j.Log4j2; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; +import pro.belbix.ethparser.model.ProfitResult; import pro.belbix.ethparser.service.ProfitService; @RestController @@ -49,7 +49,7 @@ public Double fetchProfitByVault( } @GetMapping("api/profit") - public BigDecimal calculateProfit(@RequestParam String address, @RequestParam String network) { - return profitService.calculateProfit(address, network); + public ProfitResult calculateProfit(@RequestParam String address, @RequestParam String network) { + return new ProfitResult(profitService.calculateProfit(address, network)); } } diff --git a/src/main/java/pro/belbix/ethparser/entity/profit/TokenPrice.java b/src/main/java/pro/belbix/ethparser/entity/profit/TokenPrice.java index c18f7f3b..d6b58f71 100644 --- a/src/main/java/pro/belbix/ethparser/entity/profit/TokenPrice.java +++ b/src/main/java/pro/belbix/ethparser/entity/profit/TokenPrice.java @@ -19,8 +19,4 @@ public class TokenPrice { @Id String id; Double value; - - public static String toId(String vaultAddress, String block, String network) { - return String.join("_", vaultAddress, block, network); - } } diff --git a/src/main/java/pro/belbix/ethparser/model/CovalenthqTransactionHistory.java b/src/main/java/pro/belbix/ethparser/model/CovalenthqTransactionHistory.java index 40a998d7..aa539701 100644 --- a/src/main/java/pro/belbix/ethparser/model/CovalenthqTransactionHistory.java +++ b/src/main/java/pro/belbix/ethparser/model/CovalenthqTransactionHistory.java @@ -21,12 +21,6 @@ public static class CovalenthqTransactionHistoryItems { @Data public static class CovalenthqTransactionHistoryItem { - @JsonProperty("block_signed_at") - @JsonSerialize(using = LocalDateTimeSerializer.class) - @JsonDeserialize(using = LocalDateTimeDeserializer.class) - @JsonFormat(pattern = "yyyy-MM-dd'T'HH:mm:ss'Z'") - // 2021-01-03T16:55:11Z - private LocalDateTime signedAt; @JsonProperty("block_height") private long blockHeight; @JsonProperty("tx_hash") @@ -40,6 +34,11 @@ public static class CovalenthqTransactionHistoryItem { @Data public static class CovalenthqTransactionHistoryItemLog { + @JsonProperty("block_signed_at") + @JsonSerialize(using = LocalDateTimeSerializer.class) + @JsonDeserialize(using = LocalDateTimeDeserializer.class) + @JsonFormat(pattern = "yyyy-MM-dd'T'HH:mm:ss'Z'") + private LocalDateTime signedAt; @JsonProperty("block_height") private long blockHeight; @JsonProperty("tx_hash") diff --git a/src/main/java/pro/belbix/ethparser/model/ProfitResult.java b/src/main/java/pro/belbix/ethparser/model/ProfitResult.java new file mode 100644 index 00000000..3e3dba3a --- /dev/null +++ b/src/main/java/pro/belbix/ethparser/model/ProfitResult.java @@ -0,0 +1,11 @@ +package pro.belbix.ethparser.model; + +import java.math.BigDecimal; +import lombok.AllArgsConstructor; +import lombok.Data; + +@Data +@AllArgsConstructor +public class ProfitResult { + private BigDecimal result; +} diff --git a/src/main/java/pro/belbix/ethparser/service/ProfitService.java b/src/main/java/pro/belbix/ethparser/service/ProfitService.java index 17e246a7..e93616ee 100644 --- a/src/main/java/pro/belbix/ethparser/service/ProfitService.java +++ b/src/main/java/pro/belbix/ethparser/service/ProfitService.java @@ -23,6 +23,8 @@ @Log4j2 public class ProfitService { + private final static String WITHDRAW_NAME = "Withdraw"; + private final HarvestRepository harvestRepository; private final ContractDbService contractDbService; private final EthBlockService ethBlockService; @@ -48,14 +50,24 @@ public BigDecimal calculateProfit(String address, String network) { BigDecimal totalProfit = BigDecimal.ZERO; for (CovalenthqVaultTransaction i: transactions) { - var value = i.getValue().multiply(new BigDecimal(i.getSharePrice().toString())); - if (i.getType().equals("Withdraw")) { + if (i.getContractDecimal() == 0 || i.getSharePrice().equals(BigInteger.ZERO) || i.getTokenPrice() == 0) { + log.error("Can not calculate profit, incorrect transaction : {}", i); + throw new IllegalStateException(); + } + var decimal = new BigDecimal(new BigInteger("10").pow(i.getContractDecimal()).toString()); + var value = i.getValue() + .divide(decimal) + .multiply( + new BigDecimal(i.getSharePrice().toString()).divide(decimal) + ) + .multiply(BigDecimal.valueOf(i.getTokenPrice())); + + if (i.getType().equals(WITHDRAW_NAME)) { totalProfit = totalProfit.add(value); } else { totalProfit = totalProfit.subtract(value); } } - return totalProfit; } diff --git a/src/main/java/pro/belbix/ethparser/service/SharePriceService.java b/src/main/java/pro/belbix/ethparser/service/SharePriceService.java index f7950f7b..89502709 100644 --- a/src/main/java/pro/belbix/ethparser/service/SharePriceService.java +++ b/src/main/java/pro/belbix/ethparser/service/SharePriceService.java @@ -35,6 +35,8 @@ public BigInteger getSharePrice(String vaultAddress, long block, String network) var sharePriceInt = functionsUtils.callIntByName(GET_PRICE_PER_FULL_SHARE, vaultAddress, block, network) .orElse(BigInteger.ZERO); + // TODO ps share price is zero + //0x25550cccbd68533fa04bfd3e3ac4d09f9e00fc50 if (sharePriceInt.compareTo(BigInteger.ZERO) == 0) { log.info("Share price is zero {}", id); } diff --git a/src/main/java/pro/belbix/ethparser/service/TokenPriceService.java b/src/main/java/pro/belbix/ethparser/service/TokenPriceService.java index 95eba1e1..7c54124d 100644 --- a/src/main/java/pro/belbix/ethparser/service/TokenPriceService.java +++ b/src/main/java/pro/belbix/ethparser/service/TokenPriceService.java @@ -41,7 +41,7 @@ public Double getTokenPrice(String vaultAddress, BigDecimal amount, long block, return tokenPrice.get().getValue(); } - if (ContractUtils.isPsAddress(vaultAddress)) { + if (ContractUtils.isPsAddress(vaultAddress, network)) { return priceProvider.getPriceForCoin(ContractUtils.getFarmAddress(network), block, network); } @@ -54,6 +54,7 @@ public Double getTokenPrice(String vaultAddress, BigDecimal amount, long block, if (isToken(underlyingAddress, block, network)) { price = priceProvider.getPriceForCoin(underlyingAddress, block, network); } else { + // TODO divide on decimal price = priceProvider.getLpTokenUsdPrice(underlyingAddress, amount.doubleValue(), block, network); } diff --git a/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java b/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java index 20a91d0e..5d5aa4a5 100644 --- a/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java +++ b/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java @@ -71,7 +71,7 @@ public class CovalenthqTransactionTask { @Scheduled(fixedRate = 1000 * 60 * 60 * 24) public void start() { log.info("Begin parse vault tx"); - var executor = Executors.newFixedThreadPool(10); + var executor = Executors.newFixedThreadPool(30); vaultRepository.fetchAllByNetwork(ETH_NETWORK).stream() .map(i -> CompletableFuture.runAsync(() -> getVaultTransaction(i), executor)) @@ -191,7 +191,7 @@ private CovalenthqVaultTransaction toCovalenthqVaultTransaction( transaction.setValue( StringUtils.isEmpty(value) ? BigDecimal.ZERO : new BigDecimal(value) ); - transaction.setSignedAt(transaction.getSignedAt()); + transaction.setSignedAt(item.getSignedAt()); return transaction; } catch (Exception e) { diff --git a/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstants.java b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstants.java index bee7d589..3be5d831 100644 --- a/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstants.java +++ b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstants.java @@ -219,4 +219,19 @@ public class ContractConstants { SUSHI_FACTORY_ADDRESS, 11333218 ) ); + + public static final Map> PS_ADDRESSES_BY_NETWORK = Map.of( + ETH_NETWORK, List.of( + "0x8f5adC58b32D4e5Ca02EAC0E293D35855999436C".toLowerCase(), // ST_PS + FARM_TOKEN, // FARM TOKEN + PS_ADDRESS, // PS + PS_V0_ADDRESS // PS_V0 + ), + BSC_NETWORK, List.of( + BSC_FARM_TOKEN + ), + MATIC_NETWORK, List.of( + MATIC_FARM_TOKEN + ) + ); } diff --git a/src/main/java/pro/belbix/ethparser/web3/contracts/ContractUtils.java b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractUtils.java index 33ecd6d2..d81fbdbc 100644 --- a/src/main/java/pro/belbix/ethparser/web3/contracts/ContractUtils.java +++ b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractUtils.java @@ -18,6 +18,7 @@ import static pro.belbix.ethparser.web3.contracts.ContractConstants.ORACLES_BY_FACTORY; import static pro.belbix.ethparser.web3.contracts.ContractConstants.PARSABLE_BANCOR_TRANSACTIONS; import static pro.belbix.ethparser.web3.contracts.ContractConstants.PS_ADDRESSES; +import static pro.belbix.ethparser.web3.contracts.ContractConstants.PS_ADDRESSES_BY_NETWORK; import static pro.belbix.ethparser.web3.contracts.ContractConstants.PS_V0_ADDRESS; import static pro.belbix.ethparser.web3.contracts.ContractConstants.ZERO_ADDRESS; @@ -54,6 +55,13 @@ public static boolean isPsAddress(String address) { return PS_ADDRESSES.contains(address.toLowerCase()); } + public static boolean isPsAddress(String address, String network) { + if (address == null) { + return false; + } + return PS_ADDRESSES_BY_NETWORK.get(network).contains(address.toLowerCase()); + } + public static boolean isFarmAddress(String address) { return FARM_TOKEN.equalsIgnoreCase(address) || BSC_FARM_TOKEN.equalsIgnoreCase(address) From 5aecb03dfd0da5eae3a485e1605fb81083c1f4d0 Mon Sep 17 00:00:00 2001 From: alex Date: Tue, 8 Mar 2022 02:36:28 +0300 Subject: [PATCH 22/65] Fix problem with calculate profit --- .../error/GlobalControllerAdvice.java | 22 +++++++ .../CanNotCalculateProfitException.java | 5 ++ .../exceptions/CanNotFetchPriceException.java | 12 ---- .../belbix/ethparser/model/ErrorResponse.java | 14 ++++ .../repositories/eth/VaultRepository.java | 5 +- .../ethparser/service/ProfitService.java | 66 +++++++++---------- .../ethparser/service/TokenPriceService.java | 18 ++++- .../task/CovalenthqTransactionTask.java | 3 +- .../task/UpdateUniPairContractTask.java | 14 ++-- .../web3/contracts/ContractConstants.java | 1 + 10 files changed, 102 insertions(+), 58 deletions(-) create mode 100644 src/main/java/pro/belbix/ethparser/error/GlobalControllerAdvice.java create mode 100644 src/main/java/pro/belbix/ethparser/error/exceptions/CanNotCalculateProfitException.java create mode 100644 src/main/java/pro/belbix/ethparser/model/ErrorResponse.java diff --git a/src/main/java/pro/belbix/ethparser/error/GlobalControllerAdvice.java b/src/main/java/pro/belbix/ethparser/error/GlobalControllerAdvice.java new file mode 100644 index 00000000..37dc9b80 --- /dev/null +++ b/src/main/java/pro/belbix/ethparser/error/GlobalControllerAdvice.java @@ -0,0 +1,22 @@ +package pro.belbix.ethparser.error; + +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.bind.annotation.RestControllerAdvice; +import pro.belbix.ethparser.error.exceptions.CanNotCalculateProfitException; +import pro.belbix.ethparser.model.ErrorResponse; + +@RestControllerAdvice +public class GlobalControllerAdvice { + + @ExceptionHandler(CanNotCalculateProfitException.class) + public ResponseEntity canNotCalculateProfitException() { + return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body( + ErrorResponse.builder() + .message("Can not calculate profit, try later again") + .code("500") + .build() + ); + } +} diff --git a/src/main/java/pro/belbix/ethparser/error/exceptions/CanNotCalculateProfitException.java b/src/main/java/pro/belbix/ethparser/error/exceptions/CanNotCalculateProfitException.java new file mode 100644 index 00000000..efa15088 --- /dev/null +++ b/src/main/java/pro/belbix/ethparser/error/exceptions/CanNotCalculateProfitException.java @@ -0,0 +1,5 @@ +package pro.belbix.ethparser.error.exceptions; + +public class CanNotCalculateProfitException extends RuntimeException { + +} diff --git a/src/main/java/pro/belbix/ethparser/error/exceptions/CanNotFetchPriceException.java b/src/main/java/pro/belbix/ethparser/error/exceptions/CanNotFetchPriceException.java index 7fa53e56..aee689ad 100644 --- a/src/main/java/pro/belbix/ethparser/error/exceptions/CanNotFetchPriceException.java +++ b/src/main/java/pro/belbix/ethparser/error/exceptions/CanNotFetchPriceException.java @@ -1,16 +1,4 @@ package pro.belbix.ethparser.error.exceptions; -import lombok.AccessLevel; -import lombok.Data; -import lombok.experimental.FieldDefaults; - -@Data -@FieldDefaults(level = AccessLevel.PRIVATE) public class CanNotFetchPriceException extends RuntimeException{ - String message; - - public CanNotFetchPriceException() {} - public CanNotFetchPriceException(String message) { - this.message = message; - } } diff --git a/src/main/java/pro/belbix/ethparser/model/ErrorResponse.java b/src/main/java/pro/belbix/ethparser/model/ErrorResponse.java new file mode 100644 index 00000000..c1038cf4 --- /dev/null +++ b/src/main/java/pro/belbix/ethparser/model/ErrorResponse.java @@ -0,0 +1,14 @@ +package pro.belbix.ethparser.model; + +import lombok.AccessLevel; +import lombok.Builder; +import lombok.Data; +import lombok.experimental.FieldDefaults; + +@Data +@FieldDefaults(level = AccessLevel.PRIVATE) +@Builder +public class ErrorResponse { + String message; + String code; +} diff --git a/src/main/java/pro/belbix/ethparser/repositories/eth/VaultRepository.java b/src/main/java/pro/belbix/ethparser/repositories/eth/VaultRepository.java index 8d386c57..6e512749 100644 --- a/src/main/java/pro/belbix/ethparser/repositories/eth/VaultRepository.java +++ b/src/main/java/pro/belbix/ethparser/repositories/eth/VaultRepository.java @@ -30,6 +30,7 @@ VaultEntity findFirstByContract( List fetchAllByNetwork(@Param("network") String network); @Query("select t from VaultEntity t " - + "where t.name not like 'fUni%'") - List findAllWithoutUni(); + + "left join fetch t.contract f1 " + + "where f1.network = 'eth' AND f1.name like 'V_UNI%' OR f1.name like 'V_SUSHI%'") + List findAllOnlyUniAndSushi(); } diff --git a/src/main/java/pro/belbix/ethparser/service/ProfitService.java b/src/main/java/pro/belbix/ethparser/service/ProfitService.java index e93616ee..85f7592f 100644 --- a/src/main/java/pro/belbix/ethparser/service/ProfitService.java +++ b/src/main/java/pro/belbix/ethparser/service/ProfitService.java @@ -9,10 +9,12 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import lombok.RequiredArgsConstructor; import lombok.extern.log4j.Log4j2; import org.springframework.stereotype.Service; import pro.belbix.ethparser.dto.v0.HarvestDTO; import pro.belbix.ethparser.entity.profit.CovalenthqVaultTransaction; +import pro.belbix.ethparser.error.exceptions.CanNotCalculateProfitException; import pro.belbix.ethparser.repositories.covalenthq.CovalenthqVaultTransactionRepository; import pro.belbix.ethparser.repositories.v0.HarvestRepository; import pro.belbix.ethparser.web3.EthBlockService; @@ -21,8 +23,10 @@ @Service @Log4j2 +@RequiredArgsConstructor public class ProfitService { + private final static BigInteger DEFAULT_POW = new BigInteger("10"); private final static String WITHDRAW_NAME = "Withdraw"; private final HarvestRepository harvestRepository; @@ -32,43 +36,37 @@ public class ProfitService { private final CovalenthqVaultTransactionRepository covalenthqVaultTransactionRepository; - public ProfitService(HarvestRepository harvestRepository, - ContractDbService contractDbService, - EthBlockService ethBlockService, - FunctionsUtils functionsUtils, - CovalenthqVaultTransactionRepository covalenthqVaultTransactionRepository - ) { - this.harvestRepository = harvestRepository; - this.contractDbService = contractDbService; - this.ethBlockService = ethBlockService; - this.functionsUtils = functionsUtils; - this.covalenthqVaultTransactionRepository = covalenthqVaultTransactionRepository; - } - public BigDecimal calculateProfit(String address, String network) { - var transactions = covalenthqVaultTransactionRepository.findAllByOwnerAddressAndNetwork(address, network); - BigDecimal totalProfit = BigDecimal.ZERO; - - for (CovalenthqVaultTransaction i: transactions) { - if (i.getContractDecimal() == 0 || i.getSharePrice().equals(BigInteger.ZERO) || i.getTokenPrice() == 0) { - log.error("Can not calculate profit, incorrect transaction : {}", i); - throw new IllegalStateException(); - } - var decimal = new BigDecimal(new BigInteger("10").pow(i.getContractDecimal()).toString()); - var value = i.getValue() - .divide(decimal) - .multiply( - new BigDecimal(i.getSharePrice().toString()).divide(decimal) - ) - .multiply(BigDecimal.valueOf(i.getTokenPrice())); - - if (i.getType().equals(WITHDRAW_NAME)) { - totalProfit = totalProfit.add(value); - } else { - totalProfit = totalProfit.subtract(value); + try { + var transactions = covalenthqVaultTransactionRepository.findAllByOwnerAddressAndNetwork(address, network); + BigDecimal totalProfit = BigDecimal.ZERO; + + for (CovalenthqVaultTransaction i: transactions) { + if (i.getContractDecimal() == 0 || i.getSharePrice().equals(BigInteger.ZERO) || i.getTokenPrice() == 0) { + log.error("Can not calculate profit, incorrect transaction : {}", i); + throw new CanNotCalculateProfitException(); + } + var decimal = new BigDecimal(DEFAULT_POW.pow(i.getContractDecimal()).toString()); + var value = i.getValue() + .divide(decimal) + .multiply( + new BigDecimal(i.getSharePrice().toString()).divide(decimal) + ) + .multiply(BigDecimal.valueOf(i.getTokenPrice())); + + if (i.getType().equals(WITHDRAW_NAME)) { + totalProfit = totalProfit.add(value); + } else { + totalProfit = totalProfit.subtract(value); + } } + return totalProfit; + } catch (CanNotCalculateProfitException e) { + throw e; + } catch (Exception e) { + log.error("Error during calculate profit - ", e); + throw new CanNotCalculateProfitException(); } - return totalProfit; } public Double calculationProfitForPeriod(String address, String start, String end) { diff --git a/src/main/java/pro/belbix/ethparser/service/TokenPriceService.java b/src/main/java/pro/belbix/ethparser/service/TokenPriceService.java index 7c54124d..8639f99d 100644 --- a/src/main/java/pro/belbix/ethparser/service/TokenPriceService.java +++ b/src/main/java/pro/belbix/ethparser/service/TokenPriceService.java @@ -54,8 +54,8 @@ public Double getTokenPrice(String vaultAddress, BigDecimal amount, long block, if (isToken(underlyingAddress, block, network)) { price = priceProvider.getPriceForCoin(underlyingAddress, block, network); } else { - // TODO divide on decimal - price = priceProvider.getLpTokenUsdPrice(underlyingAddress, amount.doubleValue(), block, network); + // priceProvider.getLpTokenUsdPrice return price * amount + price = priceProvider.getLpTokenUsdPrice(underlyingAddress, amount.doubleValue(), block, network) / amount.doubleValue(); } tokenPriceRepository.save(new TokenPrice(id, price)); @@ -66,7 +66,19 @@ public Double getTokenPrice(String vaultAddress, BigDecimal amount, long block, } } - private boolean isToken(String underlyingAddress, long block, String network) { + public boolean isToken(String vaultAddress, String network) { + Long latestBlock = null; + var underlyingAddress = functionsUtils.callAddressByName(UNDERLYING, vaultAddress, latestBlock, network) + .orElse(null); + + if (underlyingAddress == null) { + return false; + } + + return isToken(underlyingAddress, latestBlock, network); + } + + public boolean isToken(String underlyingAddress, Long block, String network) { return functionsUtils.callReserves(underlyingAddress, block, network) == null; } } diff --git a/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java b/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java index 5d5aa4a5..2425eb8d 100644 --- a/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java +++ b/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java @@ -4,7 +4,6 @@ import static pro.belbix.ethparser.entity.profit.CovalenthqVaultTransactionType.DEPOSIT_UNI; import static pro.belbix.ethparser.entity.profit.CovalenthqVaultTransactionType.WITHDRAW; import static pro.belbix.ethparser.entity.profit.CovalenthqVaultTransactionType.WITHDRAW_UNI; -import static pro.belbix.ethparser.service.AbiProviderService.ETH_NETWORK; import java.math.BigDecimal; import java.math.BigInteger; @@ -73,7 +72,7 @@ public void start() { log.info("Begin parse vault tx"); var executor = Executors.newFixedThreadPool(30); - vaultRepository.fetchAllByNetwork(ETH_NETWORK).stream() + vaultRepository.findAllOnlyUniAndSushi().stream() .map(i -> CompletableFuture.runAsync(() -> getVaultTransaction(i), executor)) .collect(Collectors.toList()); diff --git a/src/main/java/pro/belbix/ethparser/service/task/UpdateUniPairContractTask.java b/src/main/java/pro/belbix/ethparser/service/task/UpdateUniPairContractTask.java index 8dd745a2..805988d3 100644 --- a/src/main/java/pro/belbix/ethparser/service/task/UpdateUniPairContractTask.java +++ b/src/main/java/pro/belbix/ethparser/service/task/UpdateUniPairContractTask.java @@ -20,14 +20,18 @@ public class UpdateUniPairContractTask { @Scheduled(fixedRate = 1000 * 60 * 60 * 24) public void start() { - contractRepository.findAllUniPairContractWithoutData().stream() + contractRepository.findAllUniPairContractWithoutData() .forEach(this::fetchUniPairContact); } public void fetchUniPairContact(ContractEntity contract) { - log.info("Begin fetch uniPairsToToken for {} {}", contract.getAddress(), contract.getNetwork()); - var token = contractLoader.loadToken(contract, contract.getNetwork(), contract.getCreated()); - var uniPairsToToken = contractLoader.linkUniPairsToToken(contract.getAddress(), contract.getCreated(), token, contract.getNetwork()); - log.info("Finish fetch uniPairsToToken - {}", uniPairsToToken); + try { + log.info("Begin fetch uniPairsToToken for {} {}", contract.getAddress(), contract.getNetwork()); + var token = contractLoader.loadToken(contract, contract.getNetwork(), contract.getCreated()); + var uniPairsToToken = contractLoader.linkUniPairsToToken(contract.getAddress(), contract.getCreated(), token, contract.getNetwork()); + log.info("Finish fetch uniPairsToToken - {}", uniPairsToToken); + } catch (Exception e) { + log.error("Can not fetch uniPair token: {}", contract); + } } } diff --git a/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstants.java b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstants.java index 3be5d831..1ce7040a 100644 --- a/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstants.java +++ b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstants.java @@ -222,6 +222,7 @@ public class ContractConstants { public static final Map> PS_ADDRESSES_BY_NETWORK = Map.of( ETH_NETWORK, List.of( + "0xd3093e3efbe00f010e8f5efe3f1cb5d9b7fe0eb1".toLowerCase(), // "0x8f5adC58b32D4e5Ca02EAC0E293D35855999436C".toLowerCase(), // ST_PS FARM_TOKEN, // FARM TOKEN PS_ADDRESS, // PS From e0be29630b9aa6e178b3d40e0d4c333f267362d1 Mon Sep 17 00:00:00 2001 From: alex Date: Thu, 10 Mar 2022 03:16:22 +0300 Subject: [PATCH 23/65] Add calculate price for curve and balancer --- scripts/application.example.yml | 15 +++ scripts/run_app/run_config.yml | 4 +- .../java/pro/belbix/ethparser/AppStarter.java | 25 ++-- .../entity/contracts/TokenEntity.java | 3 + .../contracts/TokenToUniPairEntity.java | 3 + .../repositories/eth/VaultRepository.java | 4 +- .../ethparser/service/SharePriceService.java | 8 +- .../ethparser/service/TokenPriceService.java | 36 +++--- .../task/CovalenthqTransactionTask.java | 35 +++--- .../service/task/HarvestPoolInfoTask.java | 28 +++-- .../service/task/HarvestVaultInfoTask.java | 29 +++-- .../task/UpdateUniPairContractTask.java | 16 ++- .../ethparser/web3/abi/FunctionsNames.java | 4 + .../ethparser/web3/abi/FunctionsUtils.java | 97 +++++++++++++++ .../web3/contracts/ContractConstants.java | 8 ++ .../ethparser/web3/contracts/UniPairType.java | 37 ++++++ .../models/BalancerPoolTokenInfo.java | 20 +++ .../web3/contracts/models/CurveTokenInfo.java | 19 +++ .../ethparser/web3/prices/PriceProvider.java | 116 +++++++++++++++++- src/main/resources/application.yml | 14 +++ .../ethparser/web3/PriceProviderTest.java | 14 +++ src/test/resources/application.yml | 17 ++- 22 files changed, 469 insertions(+), 83 deletions(-) create mode 100644 src/main/java/pro/belbix/ethparser/web3/contracts/UniPairType.java create mode 100644 src/main/java/pro/belbix/ethparser/web3/contracts/models/BalancerPoolTokenInfo.java create mode 100644 src/main/java/pro/belbix/ethparser/web3/contracts/models/CurveTokenInfo.java diff --git a/scripts/application.example.yml b/scripts/application.example.yml index 9b03a7d9..21770161 100644 --- a/scripts/application.example.yml +++ b/scripts/application.example.yml @@ -18,3 +18,18 @@ spring: server: port: 4142 + +task: + pool: + fixedRate: 3600000 # Each hour + enable: false + vault: + fixedRate: 3600000 # Each hour + enable: false + uni-pair: + fixedRate: 86400000 # Everyday + enable: false + transaction: + max-thread-size: 30 + fixedRate: 86400000 # Everyday + enable: false diff --git a/scripts/run_app/run_config.yml b/scripts/run_app/run_config.yml index 3e4aea06..5c4a556b 100644 --- a/scripts/run_app/run_config.yml +++ b/scripts/run_app/run_config.yml @@ -1,7 +1,7 @@ ethparser: # overrideDuplicates: true - stopOnParseError: false - onlyApi: false + stopOnParseError: true + onlyApi: true # eth: # onlyParse: true # startLogBlock: 10765094 diff --git a/src/main/java/pro/belbix/ethparser/AppStarter.java b/src/main/java/pro/belbix/ethparser/AppStarter.java index 2f900e74..17ee01cf 100644 --- a/src/main/java/pro/belbix/ethparser/AppStarter.java +++ b/src/main/java/pro/belbix/ethparser/AppStarter.java @@ -1,10 +1,13 @@ package pro.belbix.ethparser; import static pro.belbix.ethparser.ws.WsService.BANCOR_TRANSACTIONS_TOPIC_NAME; +import static pro.belbix.ethparser.ws.WsService.DEPLOYER_TRANSACTIONS_TOPIC_NAME; import static pro.belbix.ethparser.ws.WsService.HARDWORK_TOPIC_NAME; import static pro.belbix.ethparser.ws.WsService.HARVEST_TRANSACTIONS_TOPIC_NAME; import static pro.belbix.ethparser.ws.WsService.IMPORTANT_EVENTS_TOPIC_NAME; import static pro.belbix.ethparser.ws.WsService.PRICES_TOPIC_NAME; +import static pro.belbix.ethparser.ws.WsService.REWARDS_TOPIC_NAME; +import static pro.belbix.ethparser.ws.WsService.TRANSFERS_TOPIC_NAME; import static pro.belbix.ethparser.ws.WsService.UNI_TRANSACTIONS_TOPIC_NAME; import java.util.Arrays; @@ -99,17 +102,17 @@ public void start() { if (conf.isTestWs()) { startFakeDataForWebSocket(ws, conf.getTestWsRate()); } else { -// startParse(bancorPriceParser, ws, BANCOR_TRANSACTIONS_TOPIC_NAME, true); -// startParse(uniswapLpLogParser, ws, UNI_TRANSACTIONS_TOPIC_NAME, true); -// startParse(vaultActionsParser, ws, HARVEST_TRANSACTIONS_TOPIC_NAME, true); -// startParse(hardWorkParser, ws, HARDWORK_TOPIC_NAME, true); -// startParse(rewardParser, ws, REWARDS_TOPIC_NAME, true); -// startParse(importantEventsParser, ws, IMPORTANT_EVENTS_TOPIC_NAME, true); -// startParse(uniToHarvestConverter, ws, HARVEST_TRANSACTIONS_TOPIC_NAME, true); -// startParse(transferParser, ws, TRANSFERS_TOPIC_NAME, true); -// startParse(priceLogParser, ws, PRICES_TOPIC_NAME, true); -// startParse(deployerTransactionsParser, ws, -// DEPLOYER_TRANSACTIONS_TOPIC_NAME, false); + startParse(bancorPriceParser, ws, BANCOR_TRANSACTIONS_TOPIC_NAME, true); + startParse(uniswapLpLogParser, ws, UNI_TRANSACTIONS_TOPIC_NAME, true); + startParse(vaultActionsParser, ws, HARVEST_TRANSACTIONS_TOPIC_NAME, true); + startParse(hardWorkParser, ws, HARDWORK_TOPIC_NAME, true); + startParse(rewardParser, ws, REWARDS_TOPIC_NAME, true); + startParse(importantEventsParser, ws, IMPORTANT_EVENTS_TOPIC_NAME, true); + startParse(uniToHarvestConverter, ws, HARVEST_TRANSACTIONS_TOPIC_NAME, true); + startParse(transferParser, ws, TRANSFERS_TOPIC_NAME, true); + startParse(priceLogParser, ws, PRICES_TOPIC_NAME, true); + startParse(deployerTransactionsParser, ws, + DEPLOYER_TRANSACTIONS_TOPIC_NAME, false); startParseBlocks(); } } diff --git a/src/main/java/pro/belbix/ethparser/entity/contracts/TokenEntity.java b/src/main/java/pro/belbix/ethparser/entity/contracts/TokenEntity.java index b07a2db5..e7cc8e97 100644 --- a/src/main/java/pro/belbix/ethparser/entity/contracts/TokenEntity.java +++ b/src/main/java/pro/belbix/ethparser/entity/contracts/TokenEntity.java @@ -2,6 +2,8 @@ import javax.persistence.Entity; import javax.persistence.FetchType; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; import javax.persistence.Id; import javax.persistence.Index; import javax.persistence.JoinColumn; @@ -19,6 +21,7 @@ public class TokenEntity { @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) private Integer id; @OneToOne(fetch = FetchType.EAGER) @JoinColumn(name = "contract", unique = true) diff --git a/src/main/java/pro/belbix/ethparser/entity/contracts/TokenToUniPairEntity.java b/src/main/java/pro/belbix/ethparser/entity/contracts/TokenToUniPairEntity.java index c666cd51..42097f7c 100644 --- a/src/main/java/pro/belbix/ethparser/entity/contracts/TokenToUniPairEntity.java +++ b/src/main/java/pro/belbix/ethparser/entity/contracts/TokenToUniPairEntity.java @@ -2,6 +2,8 @@ import javax.persistence.Entity; import javax.persistence.FetchType; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; import javax.persistence.Id; import javax.persistence.JoinColumn; import javax.persistence.ManyToOne; @@ -16,6 +18,7 @@ public class TokenToUniPairEntity { @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) private Integer id; private Long blockStart; diff --git a/src/main/java/pro/belbix/ethparser/repositories/eth/VaultRepository.java b/src/main/java/pro/belbix/ethparser/repositories/eth/VaultRepository.java index 6e512749..03cc5321 100644 --- a/src/main/java/pro/belbix/ethparser/repositories/eth/VaultRepository.java +++ b/src/main/java/pro/belbix/ethparser/repositories/eth/VaultRepository.java @@ -31,6 +31,6 @@ VaultEntity findFirstByContract( @Query("select t from VaultEntity t " + "left join fetch t.contract f1 " - + "where f1.network = 'eth' AND f1.name like 'V_UNI%' OR f1.name like 'V_SUSHI%'") - List findAllOnlyUniAndSushi(); + + "where f1.network = 'eth'") + List findOnlyEth(); } diff --git a/src/main/java/pro/belbix/ethparser/service/SharePriceService.java b/src/main/java/pro/belbix/ethparser/service/SharePriceService.java index 89502709..f624fa20 100644 --- a/src/main/java/pro/belbix/ethparser/service/SharePriceService.java +++ b/src/main/java/pro/belbix/ethparser/service/SharePriceService.java @@ -13,6 +13,7 @@ import pro.belbix.ethparser.repositories.SharePriceRepository; import pro.belbix.ethparser.utils.ProfitUtils; import pro.belbix.ethparser.web3.abi.FunctionsUtils; +import pro.belbix.ethparser.web3.contracts.ContractConstants; @Service @RequiredArgsConstructor @@ -31,14 +32,17 @@ public BigInteger getSharePrice(String vaultAddress, long block, String network) return sharePrice.get().getValue(); } + if (ContractConstants.EXCLUDE_ADDRESSES_FOR_PRICE_SHARE_BY_NETWORK.get(network).contains(vaultAddress.toLowerCase())) { + sharePriceRepository.save(new SharePrice(id, BigInteger.ONE)); + return BigInteger.ONE; + } var sharePriceInt = functionsUtils.callIntByName(GET_PRICE_PER_FULL_SHARE, vaultAddress, block, network) .orElse(BigInteger.ZERO); - // TODO ps share price is zero - //0x25550cccbd68533fa04bfd3e3ac4d09f9e00fc50 if (sharePriceInt.compareTo(BigInteger.ZERO) == 0) { log.info("Share price is zero {}", id); + sharePriceInt = BigInteger.ONE; } sharePriceRepository.save(new SharePrice(id, sharePriceInt)); diff --git a/src/main/java/pro/belbix/ethparser/service/TokenPriceService.java b/src/main/java/pro/belbix/ethparser/service/TokenPriceService.java index 8639f99d..ef06c49b 100644 --- a/src/main/java/pro/belbix/ethparser/service/TokenPriceService.java +++ b/src/main/java/pro/belbix/ethparser/service/TokenPriceService.java @@ -1,5 +1,6 @@ package pro.belbix.ethparser.service; +import static pro.belbix.ethparser.web3.abi.FunctionsNames.NAME; import static pro.belbix.ethparser.web3.abi.FunctionsNames.UNDERLYING; import java.math.BigDecimal; @@ -7,6 +8,7 @@ import lombok.RequiredArgsConstructor; import lombok.experimental.FieldDefaults; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; import org.springframework.cache.annotation.Cacheable; import org.springframework.stereotype.Service; import pro.belbix.ethparser.entity.profit.TokenPrice; @@ -15,6 +17,7 @@ import pro.belbix.ethparser.utils.ProfitUtils; import pro.belbix.ethparser.web3.abi.FunctionsUtils; import pro.belbix.ethparser.web3.contracts.ContractUtils; +import pro.belbix.ethparser.web3.contracts.UniPairType; import pro.belbix.ethparser.web3.prices.PriceProvider; @Service @@ -51,13 +54,24 @@ public Double getTokenPrice(String vaultAddress, BigDecimal amount, long block, throw new IllegalStateException(); }); - if (isToken(underlyingAddress, block, network)) { - price = priceProvider.getPriceForCoin(underlyingAddress, block, network); - } else { + var name = functionsUtils.callStrByName(NAME, underlyingAddress, block, network) + .orElse(StringUtils.EMPTY); + + if (UniPairType.isLpUniPair(name)) { // priceProvider.getLpTokenUsdPrice return price * amount price = priceProvider.getLpTokenUsdPrice(underlyingAddress, amount.doubleValue(), block, network) / amount.doubleValue(); + } else if (UniPairType.isBalancer(name)) { + price = priceProvider.getBalancerPrice(underlyingAddress, block, network); + log.info("Fetched price balancer: {} {} = {}", underlyingAddress, network, price); + // TODO Curve can not calculate price 0x29780c39164ebbd62e9ddde50c151810070140f2 + } else if (UniPairType.isCurve(name)) { + price = priceProvider.getCurvePrice(underlyingAddress, block, network); + log.info("Fetched price curve: {} {} = {}", underlyingAddress, network, price); + } else { + price = priceProvider.getPriceForCoin(underlyingAddress, block, network); } + tokenPriceRepository.save(new TokenPrice(id, price)); return price; } catch (Exception e) { @@ -65,20 +79,4 @@ public Double getTokenPrice(String vaultAddress, BigDecimal amount, long block, throw new CanNotFetchPriceException(); } } - - public boolean isToken(String vaultAddress, String network) { - Long latestBlock = null; - var underlyingAddress = functionsUtils.callAddressByName(UNDERLYING, vaultAddress, latestBlock, network) - .orElse(null); - - if (underlyingAddress == null) { - return false; - } - - return isToken(underlyingAddress, latestBlock, network); - } - - public boolean isToken(String underlyingAddress, Long block, String network) { - return functionsUtils.callReserves(underlyingAddress, block, network) == null; - } } diff --git a/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java b/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java index 2425eb8d..9cc9b1f3 100644 --- a/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java +++ b/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java @@ -16,11 +16,10 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executors; import java.util.stream.Collectors; -import lombok.AccessLevel; import lombok.RequiredArgsConstructor; -import lombok.experimental.FieldDefaults; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Value; import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Sort; import org.springframework.scheduling.annotation.Scheduled; @@ -43,7 +42,6 @@ @Service @RequiredArgsConstructor @Slf4j -@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) public class CovalenthqTransactionTask { // max block range in covalenthq private static final int MAX_BLOCK_RANGE = 1000000; @@ -57,28 +55,35 @@ public class CovalenthqTransactionTask { "0xe1fffcc4923d04b559f4d29a8bfc6cda04eb5b0d3c460751c2402c5c5cc9109c", DEPOSIT ); - SimpleDecoder simpleDecoder; - VaultRepository vaultRepository; - CovalenthqService covalenthqService; - CovalenthqVaultTransactionRepository covalenthqVaultTransactionRepository; - Web3Functions web3Functions; - SharePriceService sharePriceService; - TokenPriceService tokenPriceService; + @Value("${task.transaction.enable}") + private Boolean enable; + @Value("${task.transaction.max-thread-size}") + private Integer maxThreadSize; - // everyday - @Scheduled(fixedRate = 1000 * 60 * 60 * 24) + private final SimpleDecoder simpleDecoder; + private final VaultRepository vaultRepository; + private final CovalenthqService covalenthqService; + private final CovalenthqVaultTransactionRepository covalenthqVaultTransactionRepository; + private final Web3Functions web3Functions; + private final SharePriceService sharePriceService; + private final TokenPriceService tokenPriceService; + + + @Scheduled(fixedRateString = "${task.transaction.fixedRate}") public void start() { + if (enable == null || !enable) { + log.info("Disable CovalenthqTransactionTask"); + } log.info("Begin parse vault tx"); - var executor = Executors.newFixedThreadPool(30); + var executor = Executors.newFixedThreadPool(maxThreadSize); - vaultRepository.findAllOnlyUniAndSushi().stream() + vaultRepository.findOnlyEth().stream() .map(i -> CompletableFuture.runAsync(() -> getVaultTransaction(i), executor)) .collect(Collectors.toList()); } - // TODO add more logs, check on null, catch exceptions private void getVaultTransaction(VaultEntity vault) { try { log.info("Run getVaultTransaction for {}", vault); diff --git a/src/main/java/pro/belbix/ethparser/service/task/HarvestPoolInfoTask.java b/src/main/java/pro/belbix/ethparser/service/task/HarvestPoolInfoTask.java index 1337d237..eeda7709 100644 --- a/src/main/java/pro/belbix/ethparser/service/task/HarvestPoolInfoTask.java +++ b/src/main/java/pro/belbix/ethparser/service/task/HarvestPoolInfoTask.java @@ -9,10 +9,10 @@ import java.util.Objects; import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; -import lombok.AccessLevel; import lombok.RequiredArgsConstructor; -import lombok.experimental.FieldDefaults; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import pro.belbix.ethparser.entity.contracts.ContractEntity; import pro.belbix.ethparser.entity.contracts.PoolEntity; @@ -30,23 +30,27 @@ @Service @RequiredArgsConstructor @Slf4j -@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) public class HarvestPoolInfoTask { private static final Long INCREASE_BLOCK_WEIGHT = 1000L; - HarvestService harvestService; - ContractLoader contractLoader; - ContractRepository contractRepository; - PoolRepository poolRepository; - EthBlockService ethBlockService; - FunctionsUtils functionsUtils; - CovalenthqService covalenthqService; + @Value("${task.pool.enable}") + private Boolean enable; + private final HarvestService harvestService; + private final ContractLoader contractLoader; + private final ContractRepository contractRepository; + private final PoolRepository poolRepository; + private final EthBlockService ethBlockService; + private final FunctionsUtils functionsUtils; + private final CovalenthqService covalenthqService; - // each hour -// @Scheduled(fixedRate = 1000 * 60 * 60) + @Scheduled(fixedRateString = "${task.pool.fixedRate}") public void start() { try { + if (enable == null || !enable) { + log.info("HarvestPoolInfoTask disabled"); + return; + } var response = harvestService.getPools(); List>> poolFutures = List.of( diff --git a/src/main/java/pro/belbix/ethparser/service/task/HarvestVaultInfoTask.java b/src/main/java/pro/belbix/ethparser/service/task/HarvestVaultInfoTask.java index 68f2737a..6a86aa2f 100644 --- a/src/main/java/pro/belbix/ethparser/service/task/HarvestVaultInfoTask.java +++ b/src/main/java/pro/belbix/ethparser/service/task/HarvestVaultInfoTask.java @@ -10,10 +10,10 @@ import java.util.Objects; import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; -import lombok.AccessLevel; import lombok.RequiredArgsConstructor; -import lombok.experimental.FieldDefaults; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import pro.belbix.ethparser.entity.contracts.ContractEntity; import pro.belbix.ethparser.entity.contracts.VaultEntity; @@ -31,23 +31,28 @@ @Service @RequiredArgsConstructor @Slf4j -@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) public class HarvestVaultInfoTask { private static final Long INCREASE_BLOCK_WEIGHT = 1000L; - HarvestService harvestService; - ContractLoader contractLoader; - ContractRepository contractRepository; - VaultRepository vaultRepository; - EthBlockService ethBlockService; - FunctionsUtils functionsUtils; - CovalenthqService covalenthqService; + @Value("${task.vault.enable}") + private Boolean enable; + private final HarvestService harvestService; + private final ContractLoader contractLoader; + private final ContractRepository contractRepository; + private final VaultRepository vaultRepository; + private final EthBlockService ethBlockService; + private final FunctionsUtils functionsUtils; + private final CovalenthqService covalenthqService; - // each hour -// @Scheduled(fixedRate = 1000 * 60 * 60) + @Scheduled(fixedRateString = "${task.vault.fixedRate}") public void start() { try { + if (enable == null || !enable) { + log.info("HarvestPoolInfoTask disabled"); + return; + } + var response = harvestService.getVaults(); List>> vaultFutures = List.of( diff --git a/src/main/java/pro/belbix/ethparser/service/task/UpdateUniPairContractTask.java b/src/main/java/pro/belbix/ethparser/service/task/UpdateUniPairContractTask.java index 805988d3..5f9602b9 100644 --- a/src/main/java/pro/belbix/ethparser/service/task/UpdateUniPairContractTask.java +++ b/src/main/java/pro/belbix/ethparser/service/task/UpdateUniPairContractTask.java @@ -1,9 +1,8 @@ package pro.belbix.ethparser.service.task; -import lombok.AccessLevel; import lombok.RequiredArgsConstructor; -import lombok.experimental.FieldDefaults; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import pro.belbix.ethparser.entity.contracts.ContractEntity; @@ -13,13 +12,18 @@ @Service @RequiredArgsConstructor @Slf4j -@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) public class UpdateUniPairContractTask { - ContractRepository contractRepository; - ContractLoader contractLoader; + @Value("${task.uni-pair.enable}") + private Boolean enable; + private final ContractRepository contractRepository; + private final ContractLoader contractLoader; - @Scheduled(fixedRate = 1000 * 60 * 60 * 24) + @Scheduled(fixedRateString = "${task.uni-pair.fixedRate}") public void start() { + if (enable == null || !enable) { + log.info("UpdateUniPairContractTask disabled"); + return; + } contractRepository.findAllUniPairContractWithoutData() .forEach(this::fetchUniPairContact); } diff --git a/src/main/java/pro/belbix/ethparser/web3/abi/FunctionsNames.java b/src/main/java/pro/belbix/ethparser/web3/abi/FunctionsNames.java index 5ca00dd5..8efb9c96 100644 --- a/src/main/java/pro/belbix/ethparser/web3/abi/FunctionsNames.java +++ b/src/main/java/pro/belbix/ethparser/web3/abi/FunctionsNames.java @@ -99,4 +99,8 @@ private FunctionsNames() { public static final String CLAIMED_REWARDS = "claimedRewards"; public static final String VAULT = "vault"; public static final String RATE_BY_PATH = "rateByPath"; + public static final String GET_POOL_ID = "getPoolId"; + public static final String GET_POOL_TOKENS = "getPoolTokens"; + public static final String GET_VAULT = "getVault"; + public static final String BALANCES = "balances"; } diff --git a/src/main/java/pro/belbix/ethparser/web3/abi/FunctionsUtils.java b/src/main/java/pro/belbix/ethparser/web3/abi/FunctionsUtils.java index 9ccdd396..a86cc9a1 100644 --- a/src/main/java/pro/belbix/ethparser/web3/abi/FunctionsUtils.java +++ b/src/main/java/pro/belbix/ethparser/web3/abi/FunctionsUtils.java @@ -4,8 +4,12 @@ import static org.web3j.abi.TypeReference.makeTypeReference; import static org.web3j.protocol.core.DefaultBlockParameterName.LATEST; import static pro.belbix.ethparser.utils.Caller.silentCall; +import static pro.belbix.ethparser.web3.abi.FunctionsNames.BALANCES; import static pro.belbix.ethparser.web3.abi.FunctionsNames.BALANCE_OF; +import static pro.belbix.ethparser.web3.abi.FunctionsNames.COINS; import static pro.belbix.ethparser.web3.abi.FunctionsNames.DECIMALS; +import static pro.belbix.ethparser.web3.abi.FunctionsNames.GET_POOL_ID; +import static pro.belbix.ethparser.web3.abi.FunctionsNames.GET_POOL_TOKENS; import static pro.belbix.ethparser.web3.abi.FunctionsNames.GET_RESERVES; import static pro.belbix.ethparser.web3.abi.FunctionsNames.NAME; import static pro.belbix.ethparser.web3.abi.FunctionsNames.TOKEN0; @@ -34,6 +38,7 @@ import org.web3j.abi.datatypes.Function; import org.web3j.abi.datatypes.Type; import org.web3j.abi.datatypes.Utf8String; +import org.web3j.abi.datatypes.generated.Bytes32; import org.web3j.abi.datatypes.generated.Bytes4; import org.web3j.abi.datatypes.generated.Uint112; import org.web3j.abi.datatypes.generated.Uint256; @@ -41,12 +46,15 @@ import org.web3j.protocol.core.DefaultBlockParameter; import org.web3j.protocol.core.DefaultBlockParameterNumber; import org.web3j.tuples.generated.Tuple2; +import org.web3j.utils.Numeric; import pro.belbix.ethparser.entity.contracts.ContractEntity; import pro.belbix.ethparser.web3.MethodDecoder; import pro.belbix.ethparser.web3.Web3Functions; import pro.belbix.ethparser.web3.contracts.ContractType; import pro.belbix.ethparser.web3.contracts.ContractUtils; import pro.belbix.ethparser.web3.contracts.db.ContractDbService; +import pro.belbix.ethparser.web3.contracts.models.BalancerPoolTokenInfo; +import pro.belbix.ethparser.web3.contracts.models.CurveTokenInfo; @SuppressWarnings("rawtypes") @Service @@ -340,6 +348,95 @@ public static List> typeReferencesList(String... names) { return types; } + public Optional getPoolId(String address, Long block, String network) { + var result = web3Functions.callFunction(new Function( + GET_POOL_ID, + Collections.emptyList(), + Collections.singletonList(new TypeReference() { + })), + address, + new DefaultBlockParameterNumber(block), + network + ); + + if (result == null || result.isEmpty()) { + return Optional.empty(); + } + var bytes = (Bytes32)result.get(0); + return Optional.of(Numeric.toHexString(bytes.getValue())); + } + + public Optional getPoolTokens(String address, Long block, String network, String poolId) { + var result = web3Functions.callFunction(new Function( + GET_POOL_TOKENS, + Collections.singletonList(new Bytes32(Numeric.hexStringToByteArray(poolId))), + List.of( + new TypeReference>() { + }, + new TypeReference>() { + }, + new TypeReference() { + } + )), + address, + new DefaultBlockParameterNumber(block), + network + ); + + if (result == null || result.size() < 2) { + return Optional.empty(); + } + + return Optional.ofNullable( + BalancerPoolTokenInfo.builder() + .address( + ((List
)result.get(0).getValue()).stream().map(Address::getValue).collect(Collectors.toList()) + ) + .balances( + ((List)result.get(1).getValue()).stream() + .map(Uint256::getValue) + .collect(Collectors.toList()) + ) + .build() + ); + } + + public Optional getCurveTokenInfo(String minter, Long block, String network, long i) { + var coin = web3Functions.callFunction(new Function( + COINS, + Collections.singletonList(new Uint256(i)), + Collections.singletonList(new TypeReference
() {})), + minter, + new DefaultBlockParameterNumber(block), + network + ); + + if (coin == null || coin.isEmpty()) { + return Optional.empty(); + } + + var balance = web3Functions.callFunction(new Function( + BALANCES, + Collections.singletonList(new Uint256(i)), + Collections.singletonList(new TypeReference() {}) + ), + minter, + new DefaultBlockParameterNumber(block), + network + ); + + if (balance == null || balance.isEmpty()) { + return Optional.empty(); + } + + return Optional.ofNullable( + CurveTokenInfo.builder() + .address(((Address)coin.get(0)).getValue()) + .balance(((Uint256)balance.get(0)).getValue()) + .build() + ); + } + // ************ PRIVATE METHODS ************************** private Function findSimpleFunction(String name, String returnType) { diff --git a/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstants.java b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstants.java index 1ce7040a..015bca80 100644 --- a/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstants.java +++ b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstants.java @@ -235,4 +235,12 @@ public class ContractConstants { MATIC_FARM_TOKEN ) ); + + public static final Map> EXCLUDE_ADDRESSES_FOR_PRICE_SHARE_BY_NETWORK = Map.of( + ETH_NETWORK, List.of( + PS_ADDRESS, + "0x59258f4e15a5fc74a7284055a8094f58108dbd4f".toLowerCase() // POOL + ) + ); + } diff --git a/src/main/java/pro/belbix/ethparser/web3/contracts/UniPairType.java b/src/main/java/pro/belbix/ethparser/web3/contracts/UniPairType.java new file mode 100644 index 00000000..9f1a230a --- /dev/null +++ b/src/main/java/pro/belbix/ethparser/web3/contracts/UniPairType.java @@ -0,0 +1,37 @@ +package pro.belbix.ethparser.web3.contracts; + +import java.util.stream.Stream; +import lombok.Getter; + +@Getter +public enum UniPairType { + + // 1inch .. + ONE_INCH("1inch"), + // Balancer .. + BALANCER("Balancer"), + // Curve.fi .. + CURVE("Curve.fi"), + // SushiSwap LP Token + SUSHISWAP("SushiSwap"), + // Uniswap V2 + UNISWAP("Uniswap"); + + UniPairType(String name) { + this.name = name.toLowerCase(); + } + + private final String name; + + public static boolean isLpUniPair(String name) { + return Stream.of(ONE_INCH, SUSHISWAP, UNISWAP).anyMatch(i -> name.toLowerCase().startsWith(i.getName())); + } + + public static boolean isBalancer(String name) { + return Stream.of(BALANCER).anyMatch(i -> name.toLowerCase().startsWith(i.getName())); + } + + public static boolean isCurve(String name) { + return Stream.of(CURVE).anyMatch(i -> name.toLowerCase().startsWith(i.getName())); + } +} diff --git a/src/main/java/pro/belbix/ethparser/web3/contracts/models/BalancerPoolTokenInfo.java b/src/main/java/pro/belbix/ethparser/web3/contracts/models/BalancerPoolTokenInfo.java new file mode 100644 index 00000000..cebabf44 --- /dev/null +++ b/src/main/java/pro/belbix/ethparser/web3/contracts/models/BalancerPoolTokenInfo.java @@ -0,0 +1,20 @@ +package pro.belbix.ethparser.web3.contracts.models; + +import java.math.BigInteger; +import java.util.List; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.FieldDefaults; + +@Data +@FieldDefaults(level = AccessLevel.PRIVATE) +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class BalancerPoolTokenInfo { + List address; + List balances; +} diff --git a/src/main/java/pro/belbix/ethparser/web3/contracts/models/CurveTokenInfo.java b/src/main/java/pro/belbix/ethparser/web3/contracts/models/CurveTokenInfo.java new file mode 100644 index 00000000..ca1e28ab --- /dev/null +++ b/src/main/java/pro/belbix/ethparser/web3/contracts/models/CurveTokenInfo.java @@ -0,0 +1,19 @@ +package pro.belbix.ethparser.web3.contracts.models; + +import java.math.BigInteger; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.FieldDefaults; + +@Data +@FieldDefaults(level = AccessLevel.PRIVATE) +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class CurveTokenInfo { + String address; + BigInteger balance; +} diff --git a/src/main/java/pro/belbix/ethparser/web3/prices/PriceProvider.java b/src/main/java/pro/belbix/ethparser/web3/prices/PriceProvider.java index a24356f4..4e1b6740 100644 --- a/src/main/java/pro/belbix/ethparser/web3/prices/PriceProvider.java +++ b/src/main/java/pro/belbix/ethparser/web3/prices/PriceProvider.java @@ -1,10 +1,13 @@ package pro.belbix.ethparser.web3.prices; import static java.util.Objects.requireNonNullElse; +import static pro.belbix.ethparser.web3.abi.FunctionsNames.GET_VAULT; +import static pro.belbix.ethparser.web3.abi.FunctionsNames.MINTER; import static pro.belbix.ethparser.web3.abi.FunctionsNames.TOTAL_SUPPLY; import static pro.belbix.ethparser.web3.contracts.ContractConstants.ZERO_ADDRESS; import static pro.belbix.ethparser.web3.contracts.ContractUtils.getBaseNetworkWrappedTokenAddress; +import java.math.BigInteger; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -15,8 +18,10 @@ import org.springframework.stereotype.Service; import org.web3j.tuples.generated.Tuple2; import pro.belbix.ethparser.entity.contracts.ContractEntity; +import pro.belbix.ethparser.error.exceptions.CanNotFetchPriceException; import pro.belbix.ethparser.properties.AppProperties; import pro.belbix.ethparser.repositories.v0.PriceRepository; +import pro.belbix.ethparser.web3.abi.FunctionsNames; import pro.belbix.ethparser.web3.abi.FunctionsUtils; import pro.belbix.ethparser.web3.contracts.ContractType; import pro.belbix.ethparser.web3.contracts.ContractUtils; @@ -26,7 +31,10 @@ @Log4j2 public class PriceProvider { - private static final boolean CHECK_BLOCK_CREATED = false; + private final static int DEFAULT_CURVE_SIZE = 3; + private final static int DEFAULT_DECIMAL = 18; + private final static BigInteger DEFAULT_POW = new BigInteger("10"); + private final static boolean CHECK_BLOCK_CREATED = false; private final Map> lastPrices = new HashMap<>(); private final FunctionsUtils functionsUtils; @@ -47,6 +55,109 @@ public double getLpTokenUsdPrice(String lpAddress, double amount, long block, St return getLpTokenUsdPriceFromEth(lpAddress, amount, block, network); } + public double getBalancerPrice(String address, Long block, String network) { + var poolId = functionsUtils.getPoolId(address, block, network) + .orElseThrow(() -> { + log.error("Can not get balancer poolId for {} {}", address, network); + throw new CanNotFetchPriceException(); + }); + + var vaultAddress = functionsUtils.callAddressByName(GET_VAULT, address, block, network) + .orElseThrow(() -> { + log.error("Can not get balancer vault for {} {}", address, network); + throw new CanNotFetchPriceException(); + }); + + var poolTokenInfo = functionsUtils.getPoolTokens(vaultAddress, block, network, poolId) + .orElseThrow(() -> { + log.error("Can not get balancer poolTokenInfo for {} {}", address, network); + throw new CanNotFetchPriceException(); + }); + + var totalSupply = functionsUtils.callIntByName(TOTAL_SUPPLY, address, block, network) + .orElseThrow(() -> { + log.error("Can not get totalSupply for {} {}", address, network); + throw new CanNotFetchPriceException(); + }).doubleValue(); + + var price = 0d; + + for (int i = 0; i < poolTokenInfo.getAddress().size(); i++) { + var tokenAddress = poolTokenInfo.getAddress().get(i); + var tokenDecimal = functionsUtils.callIntByName(FunctionsNames.DECIMALS, tokenAddress, block, network) + .orElseThrow(() -> { + log.error("Can not get token decimal for {} {}", tokenAddress, network); + throw new CanNotFetchPriceException(); + }).intValue(); + + var tokenPrice = getPriceForCoin(tokenAddress, block, network); + if (tokenPrice == 0) { + log.error("Can not fetch price for balancer {} {}", address, network); + return 0; + } + + price = price + tokenPrice * normalizePrecision(poolTokenInfo.getBalances().get(i).doubleValue(), tokenDecimal); + } + + return price / totalSupply; + } + + // TODO 0xc27bfe32e0a934a12681c1b35acf0dba0e7460ba has 0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee coin address + public double getCurvePrice(String address, Long block, String network) { + var size = DEFAULT_CURVE_SIZE; + var minter = functionsUtils.callAddressByName(MINTER, address, block, network) + .orElse(null); + + if (minter == null) { + var checkAddress = functionsUtils.getCurveTokenInfo(address, block, network, 0); + if (checkAddress.isEmpty()) { + return getPriceForCoin(address, block, network); + } + size = 2; + minter = address; + } + + var decimal = functionsUtils.callIntByName(FunctionsNames.DECIMALS, address, block, network) + .orElseThrow(() -> { + log.error("Can not get decimal for {} {}", address, network); + throw new CanNotFetchPriceException(); + }).intValue(); + + var totalSupply = functionsUtils.callIntByName(TOTAL_SUPPLY, address, block, network) + .orElseThrow(() -> { + log.error("Can not get totalSupply for {} {}", address, network); + throw new CanNotFetchPriceException(); + }).doubleValue(); + + var tvl = Double.valueOf(0); + for (int i = 0; i < size; i++) { + var index = i; + var tokenInfo = functionsUtils.getCurveTokenInfo(minter, block, network, i) + .orElseThrow(() -> { + log.error("Can not get tokeInfo for {} {}, index - {}", address, network, index); + throw new CanNotFetchPriceException(); + }); + + var tokenDecimal = functionsUtils.callIntByName(FunctionsNames.DECIMALS, tokenInfo.getAddress(), block, network) + .orElseThrow(() -> { + log.error("Can not get decimal for {} {}", tokenInfo.getAddress(), network); + throw new CanNotFetchPriceException(); + }).intValue(); + + var tokenPrice = getPriceForCoin(tokenInfo.getAddress(), block, network); + + if (tokenPrice == 0) { + log.error("Can not fetch price for curve {} {}", address, network); + return 0; + } + + var balance = normalizePrecision(tokenInfo.getBalance().doubleValue(), tokenDecimal); + tvl = tvl + tokenPrice * balance / DEFAULT_POW.pow(DEFAULT_DECIMAL).doubleValue(); + } + + return tvl * DEFAULT_POW.pow(DEFAULT_DECIMAL).doubleValue() / normalizePrecision(totalSupply, decimal) ; + } + public double getLpTokenUsdPriceFromEth(String lpAddress, double amount, long block, String network) { if (appProperties.isOnlyApi()) { @@ -291,4 +402,7 @@ public boolean isDivisionSequenceSecondDividesFirst( } } + private double normalizePrecision(Double amount, int decimal) { + return amount * DEFAULT_POW.pow(DEFAULT_DECIMAL).doubleValue() / DEFAULT_POW.pow(decimal).doubleValue(); + } } diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 7b355191..db52905d 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -59,5 +59,19 @@ management: enabled: true cache.time-to-live: 10s +task: + pool: + fixedRate: 3600000 # Each hour + enable: false + vault: + fixedRate: 3600000 # Each hour + enable: false + uni-pair: + fixedRate: 86400000 # Everyday + enable: false + transaction: + max-thread-size: 30 + fixedRate: 86400000 # Everyday + enable: true diff --git a/src/test/java/pro/belbix/ethparser/web3/PriceProviderTest.java b/src/test/java/pro/belbix/ethparser/web3/PriceProviderTest.java index d9866b49..1aa720f3 100644 --- a/src/test/java/pro/belbix/ethparser/web3/PriceProviderTest.java +++ b/src/test/java/pro/belbix/ethparser/web3/PriceProviderTest.java @@ -45,4 +45,18 @@ public void priceForBNB() { double price = priceProvider.getPriceForCoin("WBNB", 6905123, BSC_NETWORK); assertEquals(numberFormat("536.67"), String.format("%.2f", price)); } + + @Test + public void priceForBalancer_80_BAL_20_WETH() { + // B-80BAL-20WETH + double price = priceProvider.getBalancerPrice("0x5c6ee304399dbdb9c8ef030ab642b10820db8f56", 14294907L, ETH_NETWORK); + assertEquals(numberFormat("28.25"), String.format("%.2f", price)); + } + + @Test + public void priceForCurve_Curvefi_USD_BTC_ETH() { + //Curve.fi USD-BTC-ETH + double price = priceProvider.getCurvePrice("0xc4AD29ba4B3c580e6D59105FFf484999997675Ff", 14354885L, ETH_NETWORK); + assertEquals(numberFormat("1459.31"), String.format("%.2f", price)); + } } diff --git a/src/test/resources/application.yml b/src/test/resources/application.yml index c333d189..ace194c9 100644 --- a/src/test/resources/application.yml +++ b/src/test/resources/application.yml @@ -51,4 +51,19 @@ logging: # only for m1 Mac users wiremock: server: - httpsPort: -1 \ No newline at end of file + httpsPort: -1 + +task: + pool: + fixedRate: 3600000 # Each hour + enable: false + vault: + fixedRate: 3600000 # Each hour + enable: false + uni-pair: + fixedRate: 86400000 # Everyday + enable: false + transaction: + max-thread-size: 30 + fixedRate: 86400000 # Everyday + enable: false From c83eb618213170bb817a527a46da6f7635513c2d Mon Sep 17 00:00:00 2001 From: alex Date: Thu, 10 Mar 2022 11:28:02 +0300 Subject: [PATCH 24/65] Change application.yml --- src/main/resources/application.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index db52905d..d36379ae 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -72,6 +72,6 @@ task: transaction: max-thread-size: 30 fixedRate: 86400000 # Everyday - enable: true + enable: false From 0539ff618b2237e24bd4e87e59dce39e71e6799f Mon Sep 17 00:00:00 2001 From: alex Date: Thu, 10 Mar 2022 12:20:07 +0300 Subject: [PATCH 25/65] Add new endpoint for calculate vault profit --- scripts/application.example.yml | 9 +++ .../controllers/ProfitController.java | 8 ++- .../CovalenthqVaultTransactionRepository.java | 5 ++ .../ethparser/service/ProfitService.java | 64 ++++++++++++------- src/main/resources/application.yml | 9 +++ 5 files changed, 72 insertions(+), 23 deletions(-) diff --git a/scripts/application.example.yml b/scripts/application.example.yml index 21770161..3e8f0021 100644 --- a/scripts/application.example.yml +++ b/scripts/application.example.yml @@ -19,6 +19,15 @@ spring: server: port: 4142 +external: + covalenthq: + url: https://api.covalenthq.com/v1/ + key: ckey_91bf3242cc5042599af1094b965 + harvest: + url: https://api-ui.harvest.finance/ + key: 41e90ced-d559-4433-b390-af424fdc76d6 + + task: pool: fixedRate: 3600000 # Each hour diff --git a/src/main/java/pro/belbix/ethparser/controllers/ProfitController.java b/src/main/java/pro/belbix/ethparser/controllers/ProfitController.java index 095b8263..d4f2d3d8 100644 --- a/src/main/java/pro/belbix/ethparser/controllers/ProfitController.java +++ b/src/main/java/pro/belbix/ethparser/controllers/ProfitController.java @@ -35,7 +35,7 @@ public Double fetchProfit( return profitService.calculationProfitForPeriod(address, start, end); } - @RequestMapping(value = "api/profit/vault", method = RequestMethod.GET) + @RequestMapping(value = "api/profit/vaults", method = RequestMethod.GET) public Double fetchProfitByVault( @RequestParam("address") @Parameter(description = "Vault address") String address, @RequestParam(value = "network", required = false, defaultValue = ETH_NETWORK) String network, @@ -52,4 +52,10 @@ public Double fetchProfitByVault( public ProfitResult calculateProfit(@RequestParam String address, @RequestParam String network) { return new ProfitResult(profitService.calculateProfit(address, network)); } + + @GetMapping("api/profit/vault") + public ProfitResult calculateVaultProfit(@RequestParam String address, @RequestParam String network, + @RequestParam long blockFrom, @RequestParam long blockTo) { + return new ProfitResult(profitService.calculateVaultProfit(address, network, blockFrom, blockTo)); + } } diff --git a/src/main/java/pro/belbix/ethparser/repositories/covalenthq/CovalenthqVaultTransactionRepository.java b/src/main/java/pro/belbix/ethparser/repositories/covalenthq/CovalenthqVaultTransactionRepository.java index 8e7bd58d..fee7047a 100644 --- a/src/main/java/pro/belbix/ethparser/repositories/covalenthq/CovalenthqVaultTransactionRepository.java +++ b/src/main/java/pro/belbix/ethparser/repositories/covalenthq/CovalenthqVaultTransactionRepository.java @@ -3,10 +3,15 @@ import java.util.List; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Query; import pro.belbix.ethparser.entity.profit.CovalenthqVaultTransaction; public interface CovalenthqVaultTransactionRepository extends JpaRepository { List findAllByNetworkAndContractAddress(String network, String contractAddress, Pageable pageable); List findAllByTransactionHashIn(List transactionHashes); List findAllByOwnerAddressAndNetwork(String ownerAddress, String network); + + @Query("select c from CovalenthqVaultTransaction c " + + "where c.network = :network AND c.contractAddress = :address AND c.block BETWEEN :blockFrom AND :blockTo") + List findAllByContractAddressAndBlockBetween(String address, String network, long blockFrom, long blockTo); } diff --git a/src/main/java/pro/belbix/ethparser/service/ProfitService.java b/src/main/java/pro/belbix/ethparser/service/ProfitService.java index 85f7592f..e07359c9 100644 --- a/src/main/java/pro/belbix/ethparser/service/ProfitService.java +++ b/src/main/java/pro/belbix/ethparser/service/ProfitService.java @@ -39,28 +39,23 @@ public class ProfitService { public BigDecimal calculateProfit(String address, String network) { try { var transactions = covalenthqVaultTransactionRepository.findAllByOwnerAddressAndNetwork(address, network); - BigDecimal totalProfit = BigDecimal.ZERO; - - for (CovalenthqVaultTransaction i: transactions) { - if (i.getContractDecimal() == 0 || i.getSharePrice().equals(BigInteger.ZERO) || i.getTokenPrice() == 0) { - log.error("Can not calculate profit, incorrect transaction : {}", i); - throw new CanNotCalculateProfitException(); - } - var decimal = new BigDecimal(DEFAULT_POW.pow(i.getContractDecimal()).toString()); - var value = i.getValue() - .divide(decimal) - .multiply( - new BigDecimal(i.getSharePrice().toString()).divide(decimal) - ) - .multiply(BigDecimal.valueOf(i.getTokenPrice())); - - if (i.getType().equals(WITHDRAW_NAME)) { - totalProfit = totalProfit.add(value); - } else { - totalProfit = totalProfit.subtract(value); - } - } - return totalProfit; + + + return calculateTxProfit(transactions); + } catch (CanNotCalculateProfitException e) { + throw e; + } catch (Exception e) { + log.error("Error during calculate profit - ", e); + throw new CanNotCalculateProfitException(); + } + } + + public BigDecimal calculateVaultProfit(String address, String network, long blockFrom, long blockTo) { + try { + var transactions = + covalenthqVaultTransactionRepository.findAllByContractAddressAndBlockBetween(address, network, blockFrom, blockTo); + + return calculateTxProfit(transactions); } catch (CanNotCalculateProfitException e) { throw e; } catch (Exception e) { @@ -185,4 +180,29 @@ private long getEndBlockNumber(List v, Long end) { return (v.get(v.size() - 1).getBlock() * end) / v.get(v.size() - 1).getBlockDate(); } + + private BigDecimal calculateTxProfit(List transactions) { + BigDecimal totalProfit = BigDecimal.ZERO; + + for (CovalenthqVaultTransaction i: transactions) { + if (i.getContractDecimal() == 0 || i.getSharePrice().equals(BigInteger.ZERO) || i.getTokenPrice() == 0) { + log.error("Can not calculate profit, incorrect transaction : {}", i); + throw new CanNotCalculateProfitException(); + } + var decimal = new BigDecimal(DEFAULT_POW.pow(i.getContractDecimal()).toString()); + var value = i.getValue() + .divide(decimal) + .multiply( + new BigDecimal(i.getSharePrice().toString()).divide(decimal) + ) + .multiply(BigDecimal.valueOf(i.getTokenPrice())); + + if (i.getType().equals(WITHDRAW_NAME)) { + totalProfit = totalProfit.add(value); + } else { + totalProfit = totalProfit.subtract(value); + } + } + return totalProfit; + } } diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index d36379ae..be960eaf 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -59,6 +59,15 @@ management: enabled: true cache.time-to-live: 10s +external: + covalenthq: + url: https://api.covalenthq.com/v1/ + key: ckey_91bf3242cc5042599af1094b965 + harvest: + url: https://api-ui.harvest.finance/ + key: 41e90ced-d559-4433-b390-af424fdc76d6 + + task: pool: fixedRate: 3600000 # Each hour From f3492e246f7b7a0ef6cf8686b295a652c859eaa0 Mon Sep 17 00:00:00 2001 From: alex Date: Fri, 11 Mar 2022 01:53:50 +0300 Subject: [PATCH 26/65] Add new vaults for calculate profit --- .../belbix/ethparser/entity/contracts/ContractEntity.java | 5 ++--- .../ethparser/repositories/eth/ContractRepository.java | 3 +++ .../belbix/ethparser/repositories/eth/VaultRepository.java | 7 +++++-- .../ethparser/service/task/CovalenthqTransactionTask.java | 2 +- .../ethparser/service/task/HarvestVaultInfoTask.java | 2 ++ .../belbix/ethparser/web3/contracts/ContractConstants.java | 4 +++- 6 files changed, 16 insertions(+), 7 deletions(-) diff --git a/src/main/java/pro/belbix/ethparser/entity/contracts/ContractEntity.java b/src/main/java/pro/belbix/ethparser/entity/contracts/ContractEntity.java index ce105f64..1f9fd928 100644 --- a/src/main/java/pro/belbix/ethparser/entity/contracts/ContractEntity.java +++ b/src/main/java/pro/belbix/ethparser/entity/contracts/ContractEntity.java @@ -1,8 +1,6 @@ package pro.belbix.ethparser.entity.contracts; import javax.persistence.Entity; -import javax.persistence.GeneratedValue; -import javax.persistence.GenerationType; import javax.persistence.Id; import javax.persistence.Index; import javax.persistence.Table; @@ -26,7 +24,8 @@ public class ContractEntity { @Id - @GeneratedValue(strategy = GenerationType.IDENTITY) +// TODO Can not save for HarvestVaultInfoTask, after return GeneratedValue annotation +// @GeneratedValue(strategy = GenerationType.IDENTITY) private Integer id; private String address; private String name; diff --git a/src/main/java/pro/belbix/ethparser/repositories/eth/ContractRepository.java b/src/main/java/pro/belbix/ethparser/repositories/eth/ContractRepository.java index 142ab8a9..992b68a6 100644 --- a/src/main/java/pro/belbix/ethparser/repositories/eth/ContractRepository.java +++ b/src/main/java/pro/belbix/ethparser/repositories/eth/ContractRepository.java @@ -65,4 +65,7 @@ List findPoolsByVaultAddress( + "left join TokenToUniPairEntity t on t.uniPair.id = u.id " + "where c.type = 2 and t.id is null") List findAllUniPairContractWithoutData(); + + @Query("select max(t.id) from ContractEntity t") + int findMaxId(); } diff --git a/src/main/java/pro/belbix/ethparser/repositories/eth/VaultRepository.java b/src/main/java/pro/belbix/ethparser/repositories/eth/VaultRepository.java index 03cc5321..3ade9688 100644 --- a/src/main/java/pro/belbix/ethparser/repositories/eth/VaultRepository.java +++ b/src/main/java/pro/belbix/ethparser/repositories/eth/VaultRepository.java @@ -31,6 +31,9 @@ VaultEntity findFirstByContract( @Query("select t from VaultEntity t " + "left join fetch t.contract f1 " - + "where f1.network = 'eth'") - List findOnlyEth(); + + "where t.name not like 'fUniV3%'") + List findNotUniV3(); + + @Query("select max(t.id) from VaultEntity t") + int findMaxId(); } diff --git a/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java b/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java index 9cc9b1f3..4b035688 100644 --- a/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java +++ b/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java @@ -78,7 +78,7 @@ public void start() { log.info("Begin parse vault tx"); var executor = Executors.newFixedThreadPool(maxThreadSize); - vaultRepository.findOnlyEth().stream() + vaultRepository.findNotUniV3().stream() .map(i -> CompletableFuture.runAsync(() -> getVaultTransaction(i), executor)) .collect(Collectors.toList()); diff --git a/src/main/java/pro/belbix/ethparser/service/task/HarvestVaultInfoTask.java b/src/main/java/pro/belbix/ethparser/service/task/HarvestVaultInfoTask.java index 6a86aa2f..b744c130 100644 --- a/src/main/java/pro/belbix/ethparser/service/task/HarvestVaultInfoTask.java +++ b/src/main/java/pro/belbix/ethparser/service/task/HarvestVaultInfoTask.java @@ -103,6 +103,8 @@ private List doTaskByAddressAndNetwork(Map Date: Sun, 13 Mar 2022 14:42:20 +0300 Subject: [PATCH 27/65] Change profit endpoint --- .../belbix/ethparser/controllers/ProfitController.java | 9 +++++++-- .../CovalenthqVaultTransactionRepository.java | 7 ++++++- .../pro/belbix/ethparser/service/ProfitService.java | 10 +++++++--- .../service/task/CovalenthqTransactionTask.java | 3 ++- 4 files changed, 22 insertions(+), 7 deletions(-) diff --git a/src/main/java/pro/belbix/ethparser/controllers/ProfitController.java b/src/main/java/pro/belbix/ethparser/controllers/ProfitController.java index d4f2d3d8..82185bae 100644 --- a/src/main/java/pro/belbix/ethparser/controllers/ProfitController.java +++ b/src/main/java/pro/belbix/ethparser/controllers/ProfitController.java @@ -49,8 +49,13 @@ public Double fetchProfitByVault( } @GetMapping("api/profit") - public ProfitResult calculateProfit(@RequestParam String address, @RequestParam String network) { - return new ProfitResult(profitService.calculateProfit(address, network)); + public ProfitResult calculateProfit( + @RequestParam String address, + @RequestParam(required = false, defaultValue = "eth") String network, + @RequestParam(required = false, defaultValue = "") String vaultAddress, + @RequestParam(required = false, defaultValue = "0") Long blockFrom, + @RequestParam(required = false, defaultValue = "0") Long blockTo) { + return new ProfitResult(profitService.calculateProfit(address, network, vaultAddress, blockFrom, blockTo)); } @GetMapping("api/profit/vault") diff --git a/src/main/java/pro/belbix/ethparser/repositories/covalenthq/CovalenthqVaultTransactionRepository.java b/src/main/java/pro/belbix/ethparser/repositories/covalenthq/CovalenthqVaultTransactionRepository.java index fee7047a..c19aa025 100644 --- a/src/main/java/pro/belbix/ethparser/repositories/covalenthq/CovalenthqVaultTransactionRepository.java +++ b/src/main/java/pro/belbix/ethparser/repositories/covalenthq/CovalenthqVaultTransactionRepository.java @@ -9,7 +9,12 @@ public interface CovalenthqVaultTransactionRepository extends JpaRepository { List findAllByNetworkAndContractAddress(String network, String contractAddress, Pageable pageable); List findAllByTransactionHashIn(List transactionHashes); - List findAllByOwnerAddressAndNetwork(String ownerAddress, String network); + @Query("select c from CovalenthqVaultTransaction c " + + "where c.network = :network AND " + + "c.contractAddress like %:address% AND " + + "c.block BETWEEN :blockFrom AND :blockTo AND " + + "c.ownerAddress = :ownerAddress") + List findAllByOwnerAddressAndNetwork(String ownerAddress, String network, String address, Long blockFrom, Long blockTo); @Query("select c from CovalenthqVaultTransaction c " + "where c.network = :network AND c.contractAddress = :address AND c.block BETWEEN :blockFrom AND :blockTo") diff --git a/src/main/java/pro/belbix/ethparser/service/ProfitService.java b/src/main/java/pro/belbix/ethparser/service/ProfitService.java index e07359c9..db11f390 100644 --- a/src/main/java/pro/belbix/ethparser/service/ProfitService.java +++ b/src/main/java/pro/belbix/ethparser/service/ProfitService.java @@ -18,6 +18,7 @@ import pro.belbix.ethparser.repositories.covalenthq.CovalenthqVaultTransactionRepository; import pro.belbix.ethparser.repositories.v0.HarvestRepository; import pro.belbix.ethparser.web3.EthBlockService; +import pro.belbix.ethparser.web3.Web3Functions; import pro.belbix.ethparser.web3.abi.FunctionsUtils; import pro.belbix.ethparser.web3.contracts.db.ContractDbService; @@ -34,12 +35,15 @@ public class ProfitService { private final EthBlockService ethBlockService; private final FunctionsUtils functionsUtils; private final CovalenthqVaultTransactionRepository covalenthqVaultTransactionRepository; + private final Web3Functions web3Functions; - public BigDecimal calculateProfit(String address, String network) { + public BigDecimal calculateProfit(String address, String network, String vault, Long blockFrom, Long blockTo) { try { - var transactions = covalenthqVaultTransactionRepository.findAllByOwnerAddressAndNetwork(address, network); - + if (blockTo == 0) { + blockTo = web3Functions.fetchCurrentBlock(network).longValue(); + } + var transactions = covalenthqVaultTransactionRepository.findAllByOwnerAddressAndNetwork(address, network, vault, blockFrom, blockTo); return calculateTxProfit(transactions); } catch (CanNotCalculateProfitException e) { diff --git a/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java b/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java index 4b035688..b6e07861 100644 --- a/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java +++ b/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java @@ -4,6 +4,7 @@ import static pro.belbix.ethparser.entity.profit.CovalenthqVaultTransactionType.DEPOSIT_UNI; import static pro.belbix.ethparser.entity.profit.CovalenthqVaultTransactionType.WITHDRAW; import static pro.belbix.ethparser.entity.profit.CovalenthqVaultTransactionType.WITHDRAW_UNI; +import static pro.belbix.ethparser.service.AbiProviderService.BSC_NETWORK; import java.math.BigDecimal; import java.math.BigInteger; @@ -78,7 +79,7 @@ public void start() { log.info("Begin parse vault tx"); var executor = Executors.newFixedThreadPool(maxThreadSize); - vaultRepository.findNotUniV3().stream() + vaultRepository.fetchAllByNetwork(BSC_NETWORK).stream() .map(i -> CompletableFuture.runAsync(() -> getVaultTransaction(i), executor)) .collect(Collectors.toList()); From 1040a16301f7da14b7f6cf7f4603691955703cfe Mon Sep 17 00:00:00 2001 From: alex Date: Tue, 15 Mar 2022 01:05:24 +0300 Subject: [PATCH 28/65] Add new endpoint for getting profit --- .../controllers/ProfitController.java | 7 +++++ .../ethparser/model/ProfitListResult.java | 31 +++++++++++++++++++ .../CovalenthqVaultTransactionRepository.java | 4 +++ .../repositories/eth/ContractRepository.java | 4 +++ .../ethparser/service/ProfitService.java | 30 ++++++++++++++++++ .../web3/contracts/db/ContractDbService.java | 3 ++ 6 files changed, 79 insertions(+) create mode 100644 src/main/java/pro/belbix/ethparser/model/ProfitListResult.java diff --git a/src/main/java/pro/belbix/ethparser/controllers/ProfitController.java b/src/main/java/pro/belbix/ethparser/controllers/ProfitController.java index 82185bae..d42e9c38 100644 --- a/src/main/java/pro/belbix/ethparser/controllers/ProfitController.java +++ b/src/main/java/pro/belbix/ethparser/controllers/ProfitController.java @@ -5,10 +5,12 @@ import io.swagger.v3.oas.annotations.Parameter; import lombok.extern.log4j.Log4j2; import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; +import pro.belbix.ethparser.model.ProfitListResult; import pro.belbix.ethparser.model.ProfitResult; import pro.belbix.ethparser.service.ProfitService; @@ -58,6 +60,11 @@ public ProfitResult calculateProfit( return new ProfitResult(profitService.calculateProfit(address, network, vaultAddress, blockFrom, blockTo)); } + @GetMapping("api/profit/{address}") + public ProfitListResult calculateProfit(@PathVariable String address) { + return profitService.calculateProfit(address); + } + @GetMapping("api/profit/vault") public ProfitResult calculateVaultProfit(@RequestParam String address, @RequestParam String network, @RequestParam long blockFrom, @RequestParam long blockTo) { diff --git a/src/main/java/pro/belbix/ethparser/model/ProfitListResult.java b/src/main/java/pro/belbix/ethparser/model/ProfitListResult.java new file mode 100644 index 00000000..09aa4cba --- /dev/null +++ b/src/main/java/pro/belbix/ethparser/model/ProfitListResult.java @@ -0,0 +1,31 @@ +package pro.belbix.ethparser.model; + +import java.math.BigDecimal; +import java.util.List; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.FieldDefaults; + +@Data +@FieldDefaults(level = AccessLevel.PRIVATE) +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class ProfitListResult { + BigDecimal totalProfit; + List items; + + @Data + @FieldDefaults(level = AccessLevel.PRIVATE) + @Builder + @AllArgsConstructor + @NoArgsConstructor + public static class ProfitListResultItem { + String name; + String contractAddress; + BigDecimal profit; + } +} diff --git a/src/main/java/pro/belbix/ethparser/repositories/covalenthq/CovalenthqVaultTransactionRepository.java b/src/main/java/pro/belbix/ethparser/repositories/covalenthq/CovalenthqVaultTransactionRepository.java index c19aa025..d27ac4e5 100644 --- a/src/main/java/pro/belbix/ethparser/repositories/covalenthq/CovalenthqVaultTransactionRepository.java +++ b/src/main/java/pro/belbix/ethparser/repositories/covalenthq/CovalenthqVaultTransactionRepository.java @@ -19,4 +19,8 @@ public interface CovalenthqVaultTransactionRepository extends JpaRepository findAllByContractAddressAndBlockBetween(String address, String network, long blockFrom, long blockTo); + + @Query("select c from CovalenthqVaultTransaction c " + + "where c.ownerAddress = :address and c.network = :network") + List findAllByOwnerAndNetwork(String address, String network); } diff --git a/src/main/java/pro/belbix/ethparser/repositories/eth/ContractRepository.java b/src/main/java/pro/belbix/ethparser/repositories/eth/ContractRepository.java index 992b68a6..64403d99 100644 --- a/src/main/java/pro/belbix/ethparser/repositories/eth/ContractRepository.java +++ b/src/main/java/pro/belbix/ethparser/repositories/eth/ContractRepository.java @@ -66,6 +66,10 @@ List findPoolsByVaultAddress( + "where c.type = 2 and t.id is null") List findAllUniPairContractWithoutData(); + @Query("select c from ContractEntity c " + + "where c.type = 0 and c.network = :network") + List findAllVaultsByNetwork(String network); + @Query("select max(t.id) from ContractEntity t") int findMaxId(); } diff --git a/src/main/java/pro/belbix/ethparser/service/ProfitService.java b/src/main/java/pro/belbix/ethparser/service/ProfitService.java index db11f390..705701c4 100644 --- a/src/main/java/pro/belbix/ethparser/service/ProfitService.java +++ b/src/main/java/pro/belbix/ethparser/service/ProfitService.java @@ -1,5 +1,6 @@ package pro.belbix.ethparser.service; +import static pro.belbix.ethparser.service.AbiProviderService.ETH_NETWORK; import static pro.belbix.ethparser.utils.CommonUtils.parseLong; import static pro.belbix.ethparser.web3.abi.FunctionsNames.BALANCE_OF; import static pro.belbix.ethparser.web3.abi.FunctionsNames.GET_PRICE_PER_FULL_SHARE; @@ -13,8 +14,11 @@ import lombok.extern.log4j.Log4j2; import org.springframework.stereotype.Service; import pro.belbix.ethparser.dto.v0.HarvestDTO; +import pro.belbix.ethparser.entity.contracts.ContractEntity; import pro.belbix.ethparser.entity.profit.CovalenthqVaultTransaction; import pro.belbix.ethparser.error.exceptions.CanNotCalculateProfitException; +import pro.belbix.ethparser.model.ProfitListResult; +import pro.belbix.ethparser.model.ProfitListResult.ProfitListResultItem; import pro.belbix.ethparser.repositories.covalenthq.CovalenthqVaultTransactionRepository; import pro.belbix.ethparser.repositories.v0.HarvestRepository; import pro.belbix.ethparser.web3.EthBlockService; @@ -38,6 +42,16 @@ public class ProfitService { private final Web3Functions web3Functions; + public ProfitListResult calculateProfit(String address) { + var transactions = covalenthqVaultTransactionRepository.findAllByOwnerAndNetwork(address, ETH_NETWORK); + var contracts = contractDbService.findAllVaultsByNetwork(ETH_NETWORK); + + return ProfitListResult.builder() + .totalProfit(calculateTxProfit(transactions)) + .items(calculateProfitByVaults(transactions, contracts)) + .build(); + } + public BigDecimal calculateProfit(String address, String network, String vault, Long blockFrom, Long blockTo) { try { if (blockTo == 0) { @@ -209,4 +223,20 @@ private BigDecimal calculateTxProfit(List transactio } return totalProfit; } + + private List calculateProfitByVaults(List transactions, List contracts) { + return contracts.stream() + .map(i -> { + var txByContract = transactions.stream() + .filter(tx -> tx.getContractAddress().equalsIgnoreCase(i.getAddress())) + .collect(Collectors.toList()); + + return ProfitListResultItem.builder() + .contractAddress(i.getAddress()) + .name(i.getName()) + .profit(calculateTxProfit(txByContract)) + .build(); + }) + .collect(Collectors.toList()); + } } diff --git a/src/main/java/pro/belbix/ethparser/web3/contracts/db/ContractDbService.java b/src/main/java/pro/belbix/ethparser/web3/contracts/db/ContractDbService.java index 9f5fa6ff..98475f4d 100644 --- a/src/main/java/pro/belbix/ethparser/web3/contracts/db/ContractDbService.java +++ b/src/main/java/pro/belbix/ethparser/web3/contracts/db/ContractDbService.java @@ -257,4 +257,7 @@ public Optional getBaseContractForNetwork(String network) { ContractUtils.getBaseNetworkWrappedTokenAddress(network), network); } + public List findAllVaultsByNetwork(String network) { + return contractRepository.findAllVaultsByNetwork(network); + } } From 56e5f576288745394f91773a5796521c406a855e Mon Sep 17 00:00:00 2001 From: alex Date: Wed, 16 Mar 2022 02:51:44 +0300 Subject: [PATCH 29/65] Change price calculate and add network to endpoint --- .../belbix/ethparser/controllers/ProfitController.java | 4 ++-- .../ethparser/repositories/eth/VaultRepository.java | 4 ++-- .../java/pro/belbix/ethparser/service/ProfitService.java | 4 ++-- .../service/task/CovalenthqTransactionTask.java | 6 ++++-- .../service/task/UpdateUniPairContractTask.java | 1 + .../pro/belbix/ethparser/web3/contracts/UniPairType.java | 9 ++++++++- 6 files changed, 19 insertions(+), 9 deletions(-) diff --git a/src/main/java/pro/belbix/ethparser/controllers/ProfitController.java b/src/main/java/pro/belbix/ethparser/controllers/ProfitController.java index d42e9c38..62f436ef 100644 --- a/src/main/java/pro/belbix/ethparser/controllers/ProfitController.java +++ b/src/main/java/pro/belbix/ethparser/controllers/ProfitController.java @@ -61,8 +61,8 @@ public ProfitResult calculateProfit( } @GetMapping("api/profit/{address}") - public ProfitListResult calculateProfit(@PathVariable String address) { - return profitService.calculateProfit(address); + public ProfitListResult calculateProfit(@PathVariable String address, @RequestParam(required = false, defaultValue = "eth") String network) { + return profitService.calculateProfit(address, network); } @GetMapping("api/profit/vault") diff --git a/src/main/java/pro/belbix/ethparser/repositories/eth/VaultRepository.java b/src/main/java/pro/belbix/ethparser/repositories/eth/VaultRepository.java index 3ade9688..2bf99ddf 100644 --- a/src/main/java/pro/belbix/ethparser/repositories/eth/VaultRepository.java +++ b/src/main/java/pro/belbix/ethparser/repositories/eth/VaultRepository.java @@ -31,8 +31,8 @@ VaultEntity findFirstByContract( @Query("select t from VaultEntity t " + "left join fetch t.contract f1 " - + "where t.name not like 'fUniV3%'") - List findNotUniV3(); + + "where f1.network = :network") + List findAllByNetwork(String network); @Query("select max(t.id) from VaultEntity t") int findMaxId(); diff --git a/src/main/java/pro/belbix/ethparser/service/ProfitService.java b/src/main/java/pro/belbix/ethparser/service/ProfitService.java index 705701c4..6fc78b4c 100644 --- a/src/main/java/pro/belbix/ethparser/service/ProfitService.java +++ b/src/main/java/pro/belbix/ethparser/service/ProfitService.java @@ -42,8 +42,8 @@ public class ProfitService { private final Web3Functions web3Functions; - public ProfitListResult calculateProfit(String address) { - var transactions = covalenthqVaultTransactionRepository.findAllByOwnerAndNetwork(address, ETH_NETWORK); + public ProfitListResult calculateProfit(String address, String network) { + var transactions = covalenthqVaultTransactionRepository.findAllByOwnerAndNetwork(address, network); var contracts = contractDbService.findAllVaultsByNetwork(ETH_NETWORK); return ProfitListResult.builder() diff --git a/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java b/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java index b6e07861..65a94e0a 100644 --- a/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java +++ b/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java @@ -75,11 +75,12 @@ public class CovalenthqTransactionTask { public void start() { if (enable == null || !enable) { log.info("Disable CovalenthqTransactionTask"); + return; } log.info("Begin parse vault tx"); var executor = Executors.newFixedThreadPool(maxThreadSize); - vaultRepository.fetchAllByNetwork(BSC_NETWORK).stream() + vaultRepository.findAllByNetwork(BSC_NETWORK).stream() .map(i -> CompletableFuture.runAsync(() -> getVaultTransaction(i), executor)) .collect(Collectors.toList()); @@ -201,7 +202,8 @@ private CovalenthqVaultTransaction toCovalenthqVaultTransaction( return transaction; } catch (Exception e) { log.error("Can not parse covalenthq log: {}", item, e); - return null; + throw new CanNotFetchPriceException(); +// return null; } } diff --git a/src/main/java/pro/belbix/ethparser/service/task/UpdateUniPairContractTask.java b/src/main/java/pro/belbix/ethparser/service/task/UpdateUniPairContractTask.java index 5f9602b9..0193a2a8 100644 --- a/src/main/java/pro/belbix/ethparser/service/task/UpdateUniPairContractTask.java +++ b/src/main/java/pro/belbix/ethparser/service/task/UpdateUniPairContractTask.java @@ -24,6 +24,7 @@ public void start() { log.info("UpdateUniPairContractTask disabled"); return; } + log.info("Start UpdateUniPairContractTask"); contractRepository.findAllUniPairContractWithoutData() .forEach(this::fetchUniPairContact); } diff --git a/src/main/java/pro/belbix/ethparser/web3/contracts/UniPairType.java b/src/main/java/pro/belbix/ethparser/web3/contracts/UniPairType.java index 9f1a230a..b49b6832 100644 --- a/src/main/java/pro/belbix/ethparser/web3/contracts/UniPairType.java +++ b/src/main/java/pro/belbix/ethparser/web3/contracts/UniPairType.java @@ -14,6 +14,10 @@ public enum UniPairType { CURVE("Curve.fi"), // SushiSwap LP Token SUSHISWAP("SushiSwap"), + // Pancake .. + PANCACKE("Pancake"), + // PancakeSwap .. + PANCACKE_SWAP("PancakeSwap"), // Uniswap V2 UNISWAP("Uniswap"); @@ -24,7 +28,10 @@ public enum UniPairType { private final String name; public static boolean isLpUniPair(String name) { - return Stream.of(ONE_INCH, SUSHISWAP, UNISWAP).anyMatch(i -> name.toLowerCase().startsWith(i.getName())); + if (Stream.of(PANCACKE_SWAP).anyMatch(i -> name.toLowerCase().startsWith(i.getName()))) { + return false; + } + return Stream.of(ONE_INCH, SUSHISWAP, UNISWAP, PANCACKE).anyMatch(i -> name.toLowerCase().startsWith(i.getName())); } public static boolean isBalancer(String name) { From bb68c683a7778e0f59e93b2c0c3f29efc611b740 Mon Sep 17 00:00:00 2001 From: alex Date: Fri, 18 Mar 2022 01:14:36 +0300 Subject: [PATCH 30/65] Fix problem with matic parse transactions --- .../ethparser/service/ProfitService.java | 3 +-- .../task/CovalenthqTransactionTask.java | 11 +++++++--- .../belbix/ethparser/web3/MethodDecoder.java | 21 +++++++++++++------ .../belbix/ethparser/web3/SimpleDecoder.java | 2 +- .../ethparser/web3/abi/FunctionsUtils.java | 8 +++++++ .../web3/contracts/ContractConstants.java | 20 +++++++++++++++++- .../ethparser/web3/contracts/UniPairType.java | 4 +++- .../ethparser/web3/prices/PriceProvider.java | 10 ++++++++- 8 files changed, 64 insertions(+), 15 deletions(-) diff --git a/src/main/java/pro/belbix/ethparser/service/ProfitService.java b/src/main/java/pro/belbix/ethparser/service/ProfitService.java index 6fc78b4c..7de56887 100644 --- a/src/main/java/pro/belbix/ethparser/service/ProfitService.java +++ b/src/main/java/pro/belbix/ethparser/service/ProfitService.java @@ -1,6 +1,5 @@ package pro.belbix.ethparser.service; -import static pro.belbix.ethparser.service.AbiProviderService.ETH_NETWORK; import static pro.belbix.ethparser.utils.CommonUtils.parseLong; import static pro.belbix.ethparser.web3.abi.FunctionsNames.BALANCE_OF; import static pro.belbix.ethparser.web3.abi.FunctionsNames.GET_PRICE_PER_FULL_SHARE; @@ -44,7 +43,7 @@ public class ProfitService { public ProfitListResult calculateProfit(String address, String network) { var transactions = covalenthqVaultTransactionRepository.findAllByOwnerAndNetwork(address, network); - var contracts = contractDbService.findAllVaultsByNetwork(ETH_NETWORK); + var contracts = contractDbService.findAllVaultsByNetwork(network); return ProfitListResult.builder() .totalProfit(calculateTxProfit(transactions)) diff --git a/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java b/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java index 65a94e0a..28b96e25 100644 --- a/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java +++ b/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java @@ -4,7 +4,7 @@ import static pro.belbix.ethparser.entity.profit.CovalenthqVaultTransactionType.DEPOSIT_UNI; import static pro.belbix.ethparser.entity.profit.CovalenthqVaultTransactionType.WITHDRAW; import static pro.belbix.ethparser.entity.profit.CovalenthqVaultTransactionType.WITHDRAW_UNI; -import static pro.belbix.ethparser.service.AbiProviderService.BSC_NETWORK; +import static pro.belbix.ethparser.service.AbiProviderService.MATIC_NETWORK; import java.math.BigDecimal; import java.math.BigInteger; @@ -39,6 +39,7 @@ import pro.belbix.ethparser.service.external.CovalenthqService; import pro.belbix.ethparser.web3.SimpleDecoder; import pro.belbix.ethparser.web3.Web3Functions; +import pro.belbix.ethparser.web3.abi.FunctionsUtils; @Service @RequiredArgsConstructor @@ -69,6 +70,7 @@ public class CovalenthqTransactionTask { private final Web3Functions web3Functions; private final SharePriceService sharePriceService; private final TokenPriceService tokenPriceService; + private final FunctionsUtils functionsUtils; @Scheduled(fixedRateString = "${task.transaction.fixedRate}") @@ -80,7 +82,7 @@ public void start() { log.info("Begin parse vault tx"); var executor = Executors.newFixedThreadPool(maxThreadSize); - vaultRepository.findAllByNetwork(BSC_NETWORK).stream() + vaultRepository.findAllByNetwork(MATIC_NETWORK).stream() .map(i -> CompletableFuture.runAsync(() -> getVaultTransaction(i), executor)) .collect(Collectors.toList()); @@ -186,11 +188,14 @@ private CovalenthqVaultTransaction toCovalenthqVaultTransaction( throw new IllegalStateException(); }); var value = getParamValue(item, covalenthqType.value); + var decimal = item.getContractDecimal() == 0 + ? functionsUtils.getDecimal(contractAddress, network) + : item.getContractDecimal(); transaction.setNetwork(network); transaction.setBlock(item.getBlockHeight()); transaction.setTransactionHash(item.getTransactionHash()); - transaction.setContractDecimal(item.getContractDecimal()); + transaction.setContractDecimal(decimal); transaction.setContractAddress(contractAddress); transaction.setType(covalenthqType.type); transaction.setOwnerAddress(getParamValue(item, covalenthqType.address)); diff --git a/src/main/java/pro/belbix/ethparser/web3/MethodDecoder.java b/src/main/java/pro/belbix/ethparser/web3/MethodDecoder.java index e495102f..19031ae1 100644 --- a/src/main/java/pro/belbix/ethparser/web3/MethodDecoder.java +++ b/src/main/java/pro/belbix/ethparser/web3/MethodDecoder.java @@ -11,6 +11,7 @@ import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; +import lombok.extern.log4j.Log4j2; import org.web3j.abi.FunctionReturnDecoder; import org.web3j.abi.TypeReference; import org.web3j.abi.datatypes.Address; @@ -32,6 +33,7 @@ import pro.belbix.ethparser.web3.abi.CommonMethods; @SuppressWarnings({"rawtypes", "unchecked"}) +@Log4j2 public abstract class MethodDecoder { protected Map>> parametersByMethodId = new HashMap<>(); @@ -88,18 +90,25 @@ public static List extractLogIndexedValues( } List> indexedParameters = getIndexedParameters(parameters); for (int i = 0; i < indexedParameters.size(); i++) { - var index = topics.size() > i + 1 - ? i + 1 - : i; - - // TODO in some case get index out of - String topic = topics.get(index); + String topic = topics.get(i + 1); Type value = decodeIndexedValue(topic, indexedParameters.get(i)); indexedValues.add(value); } indexedValues.addAll(nonIndexedValues); return indexedValues; } + public static List extractLogIndexedValuesWrapped( + List topics, + String data, + List> parameters) { + try { + return extractLogIndexedValues(topics, data, parameters); + } catch (Exception e) { + log.error("Get error during parse log: {}", e.getMessage()); + // in some cases parameters in data + return FunctionReturnDecoder.decode(data, parameters); + } + } public static List> getNonIndexedParameters( List> parameters) { diff --git a/src/main/java/pro/belbix/ethparser/web3/SimpleDecoder.java b/src/main/java/pro/belbix/ethparser/web3/SimpleDecoder.java index 95784310..b74a97bc 100644 --- a/src/main/java/pro/belbix/ethparser/web3/SimpleDecoder.java +++ b/src/main/java/pro/belbix/ethparser/web3/SimpleDecoder.java @@ -18,7 +18,7 @@ public Optional> decodeEthLog(Log ethLog) { return parseMethodId(ethLog) .flatMap(this::findParameters) .map(parameters -> - extractLogIndexedValues(ethLog.getTopics(), ethLog.getData(), parameters)); + extractLogIndexedValuesWrapped(ethLog.getTopics(), ethLog.getData(), parameters)); } @SuppressWarnings("rawtypes") diff --git a/src/main/java/pro/belbix/ethparser/web3/abi/FunctionsUtils.java b/src/main/java/pro/belbix/ethparser/web3/abi/FunctionsUtils.java index a86cc9a1..2f4d146f 100644 --- a/src/main/java/pro/belbix/ethparser/web3/abi/FunctionsUtils.java +++ b/src/main/java/pro/belbix/ethparser/web3/abi/FunctionsUtils.java @@ -30,6 +30,7 @@ import java.util.stream.Collectors; import lombok.extern.log4j.Log4j2; import org.apache.commons.lang3.StringUtils; +import org.springframework.cache.annotation.Cacheable; import org.springframework.stereotype.Service; import org.web3j.abi.TypeReference; import org.web3j.abi.datatypes.Address; @@ -71,6 +72,7 @@ public class FunctionsUtils { private final Map functionsCache = new HashMap<>(); private final static String EXCLUDE_ONEINCH = "1inch"; private final static String EXCLUDE_KYBER = "Kyber"; + private final static BigInteger DEFAULT_DECIMAL = BigInteger.valueOf(18); private final Web3Functions web3Functions; private final ContractDbService contractDbService; @@ -96,6 +98,12 @@ public Tuple2 callReserves( } } + @Cacheable("decimal_latest_block") + public int getDecimal(String address, String network) { + return callIntByName(FunctionsNames.DECIMALS, address, null, network) + .orElse(DEFAULT_DECIMAL).intValue(); + } + private Tuple2 callOneInchReserves(String lpAddress, Long block, String network) { String coin0 = callAddressByName(TOKEN0, lpAddress, block, network) .orElseThrow(() -> new IllegalStateException("Error get token0 for " + lpAddress)); diff --git a/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstants.java b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstants.java index 160616ad..a0a977c4 100644 --- a/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstants.java +++ b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstants.java @@ -151,13 +151,15 @@ public class ContractConstants { PS_V0_ADDRESS // PS_V0 ); + // TODO separate by networks public static final Set ONE_DOLLAR_TOKENS = Set.of( "0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48".toLowerCase(), //USDC "0xe9e7cea3dedca5984780bafc599bd69add087d56".toLowerCase(), //BUSD "0xdAC17F958D2ee523a2206206994597C13D831ec7".toLowerCase(), //USDT "0x0000000000085d4780B73119b644AE5ecd22b376".toLowerCase(), //TUSD "0x2791bca1f2de4661ed88a30c99a7a9449aa84174".toLowerCase(), //matic USDC - "0xc2132d05d31c914a87c6611c10748aeb04b58e8f".toLowerCase() //matic USDT + "0xc2132d05d31c914a87c6611c10748aeb04b58e8f".toLowerCase(), //matic USDT + "0xE840B73E5287865EEc17d250bFb1536704B43B21".toLowerCase() //matic mUSD ); //Key tokens are used to find liquidity for any given token on Uni, Sushi and Curve. @@ -245,4 +247,20 @@ public class ContractConstants { MATIC_NETWORK, List.of() ); + public static final Map> EXCLUDE_JARVIS_STABLECOIN = Map.of( + ETH_NETWORK, List.of(), + BSC_NETWORK, List.of(), + MATIC_NETWORK, List.of( + "0x8ca194A3b22077359b5732DE53373D4afC11DeE3".toLowerCase(), // jCAD + "0x8343091F2499FD4b6174A46D067A920a3b851FF9".toLowerCase() // jJPY + ) + ); + + public static final Map> IS_NOT_AVAILABLE_IN_ORACLE = Map.of( + ETH_NETWORK, List.of(), + BSC_NETWORK, List.of(), + MATIC_NETWORK, List.of( + "0x32d8513eDDa5AEf930080F15270984A043933A95".toLowerCase() // KyberDMM LP jCAD+CADC-f-QUI-MAR22 + ) + ); } diff --git a/src/main/java/pro/belbix/ethparser/web3/contracts/UniPairType.java b/src/main/java/pro/belbix/ethparser/web3/contracts/UniPairType.java index b49b6832..b10177f6 100644 --- a/src/main/java/pro/belbix/ethparser/web3/contracts/UniPairType.java +++ b/src/main/java/pro/belbix/ethparser/web3/contracts/UniPairType.java @@ -18,6 +18,8 @@ public enum UniPairType { PANCACKE("Pancake"), // PancakeSwap .. PANCACKE_SWAP("PancakeSwap"), + // Kyber + KYBER("Kyber"), // Uniswap V2 UNISWAP("Uniswap"); @@ -31,7 +33,7 @@ public static boolean isLpUniPair(String name) { if (Stream.of(PANCACKE_SWAP).anyMatch(i -> name.toLowerCase().startsWith(i.getName()))) { return false; } - return Stream.of(ONE_INCH, SUSHISWAP, UNISWAP, PANCACKE).anyMatch(i -> name.toLowerCase().startsWith(i.getName())); + return Stream.of(ONE_INCH, SUSHISWAP, UNISWAP, PANCACKE, KYBER).anyMatch(i -> name.toLowerCase().startsWith(i.getName())); } public static boolean isBalancer(String name) { diff --git a/src/main/java/pro/belbix/ethparser/web3/prices/PriceProvider.java b/src/main/java/pro/belbix/ethparser/web3/prices/PriceProvider.java index 4e1b6740..d4ca1a73 100644 --- a/src/main/java/pro/belbix/ethparser/web3/prices/PriceProvider.java +++ b/src/main/java/pro/belbix/ethparser/web3/prices/PriceProvider.java @@ -4,6 +4,8 @@ import static pro.belbix.ethparser.web3.abi.FunctionsNames.GET_VAULT; import static pro.belbix.ethparser.web3.abi.FunctionsNames.MINTER; import static pro.belbix.ethparser.web3.abi.FunctionsNames.TOTAL_SUPPLY; +import static pro.belbix.ethparser.web3.contracts.ContractConstants.EXCLUDE_JARVIS_STABLECOIN; +import static pro.belbix.ethparser.web3.contracts.ContractConstants.IS_NOT_AVAILABLE_IN_ORACLE; import static pro.belbix.ethparser.web3.contracts.ContractConstants.ZERO_ADDRESS; import static pro.belbix.ethparser.web3.contracts.ContractUtils.getBaseNetworkWrappedTokenAddress; @@ -31,6 +33,7 @@ @Log4j2 public class PriceProvider { + private final static Double DEFAULT_RETURN_PRICE = 1D; private final static int DEFAULT_CURVE_SIZE = 3; private final static int DEFAULT_DECIMAL = 18; private final static BigInteger DEFAULT_POW = new BigInteger("10"); @@ -164,7 +167,7 @@ public double getLpTokenUsdPriceFromEth(String lpAddress, double amount, long bl return 0.0; } - if (PriceOracle.isAvailable(block, network)) { + if (PriceOracle.isAvailable(block, network) && IS_NOT_AVAILABLE_IN_ORACLE.get(network).stream().noneMatch(i -> i.equalsIgnoreCase(lpAddress))) { return amount * priceOracle.getPriceForCoinOnChain(lpAddress, block, network); } log.info("Oracle not deployed yet, use direct calculation for prices"); @@ -268,6 +271,11 @@ private double getPriceForCoinFromEth(String address, Long block, String network if (appProperties.isOnlyApi()) { return 0.0; } + + if (EXCLUDE_JARVIS_STABLECOIN.get(network).stream().anyMatch(i -> i.equals(address.toLowerCase()))) { + return DEFAULT_RETURN_PRICE; + } + if (PriceOracle.isAvailable(block, network)) { return priceOracle.getPriceForCoinOnChain(address, block, network); } From 355cd80b270e4b52393380e24d954f885e8112f6 Mon Sep 17 00:00:00 2001 From: alex Date: Sat, 19 Mar 2022 03:18:15 +0300 Subject: [PATCH 31/65] Fix problem with calculate price for matic vaults --- .../ethparser/web3/abi/FunctionsNames.java | 2 + .../ethparser/web3/abi/FunctionsUtils.java | 82 ++++++++++++++++++- .../ethparser/web3/prices/PriceOracle.java | 2 +- .../ethparser/web3/prices/PriceProvider.java | 22 +++-- 4 files changed, 100 insertions(+), 8 deletions(-) diff --git a/src/main/java/pro/belbix/ethparser/web3/abi/FunctionsNames.java b/src/main/java/pro/belbix/ethparser/web3/abi/FunctionsNames.java index 8efb9c96..7d7e430d 100644 --- a/src/main/java/pro/belbix/ethparser/web3/abi/FunctionsNames.java +++ b/src/main/java/pro/belbix/ethparser/web3/abi/FunctionsNames.java @@ -103,4 +103,6 @@ private FunctionsNames() { public static final String GET_POOL_TOKENS = "getPoolTokens"; public static final String GET_VAULT = "getVault"; public static final String BALANCES = "balances"; + public static final String CHECKING_PRICING_TOKEN = "checkPricingToken"; + public static final String GET_BALANCES = "get_balances"; } diff --git a/src/main/java/pro/belbix/ethparser/web3/abi/FunctionsUtils.java b/src/main/java/pro/belbix/ethparser/web3/abi/FunctionsUtils.java index 2f4d146f..7801eee9 100644 --- a/src/main/java/pro/belbix/ethparser/web3/abi/FunctionsUtils.java +++ b/src/main/java/pro/belbix/ethparser/web3/abi/FunctionsUtils.java @@ -6,6 +6,7 @@ import static pro.belbix.ethparser.utils.Caller.silentCall; import static pro.belbix.ethparser.web3.abi.FunctionsNames.BALANCES; import static pro.belbix.ethparser.web3.abi.FunctionsNames.BALANCE_OF; +import static pro.belbix.ethparser.web3.abi.FunctionsNames.CHECKING_PRICING_TOKEN; import static pro.belbix.ethparser.web3.abi.FunctionsNames.COINS; import static pro.belbix.ethparser.web3.abi.FunctionsNames.DECIMALS; import static pro.belbix.ethparser.web3.abi.FunctionsNames.GET_POOL_ID; @@ -94,7 +95,7 @@ public Tuple2 callReserves( if (lpName.startsWith(EXCLUDE_ONEINCH)) { return callOneInchReserves(lpAddress, block, network); } else { - return callUniReserves(lpAddress, block, network, getTypeReferenceForGetReserves(lpName)); + return callUniReservesOrElseGetFromBlockchain(lpAddress, block, network, lpName); } } @@ -104,6 +105,44 @@ public int getDecimal(String address, String network) { .orElse(DEFAULT_DECIMAL).intValue(); } + @Cacheable("uni_tokens_latest_block") + public Tuple2 callTokensForSwapPlatform(String address, String network) { + String token0 = callAddressByName(TOKEN0, address, null, network) + .orElseThrow(() -> new IllegalStateException("Error get token0 for " + address)); + String token1 = callAddressByName(TOKEN1, address, null, network) + .orElseThrow(() -> new IllegalStateException("Error get token1 for " + address)); + + return new Tuple2<>(token0, token1); + } + + @Cacheable("uni_reserves") + public Tuple2 callUniReservesForSwapPlatform(String address, Long block, String network, List> typeReferences) { + var types = web3Functions.callFunction(new Function( + GET_RESERVES, + Collections.emptyList(), + typeReferences), address, resolveBlock(block), network); + + if (types == null || types.size() < typeReferences.size()) { + log.error("Wrong values for " + address); + return null; + } + + var tokens = callTokensForSwapPlatform(address, network); + var v1 = (BigInteger) types.get(0).getValue(); + var v2 = (BigInteger) types.get(1).getValue(); + + return new Tuple2<>( + parseAmount(v1, tokens.component1(), network), + parseAmount(v2, tokens.component2(), network) + ); + } + + @Cacheable("can_get_token_price") + public boolean canGetTokenPrice(String tokenAddress, String oracleAddress, Long block, String network) { + return callBoolByNameWithAddressArg(CHECKING_PRICING_TOKEN, tokenAddress, oracleAddress, block, network) + .orElseThrow(() -> new IllegalStateException("Can not call checkPricingToken for address: " + tokenAddress)); + } + private Tuple2 callOneInchReserves(String lpAddress, Long block, String network) { String coin0 = callAddressByName(TOKEN0, lpAddress, block, network) .orElseThrow(() -> new IllegalStateException("Error get token0 for " + lpAddress)); @@ -149,6 +188,15 @@ private Tuple2 callUniReserves(String lpAddress, Long block, Str ); } + private Tuple2 callUniReservesOrElseGetFromBlockchain(String lpAddress, Long block, String network, String lpName) { + var types = getTypeReferenceForGetReserves(lpName); + try { + return callUniReserves(lpAddress, block, network, types); + } catch (IllegalStateException e) { + return callUniReservesForSwapPlatform(lpAddress, block, network, types); + } + } + public double fetchUint256Field( String functionName, String address, @@ -409,6 +457,38 @@ public Optional getPoolTokens(String address, Long block, ); } + @Cacheable("address_name") + public String getName(String address, String network) { + return callStrByName(NAME, address, null, network) + .orElse(StringUtils.EMPTY); + } + + @Cacheable("curve_vault_size") + public int getCurveVaultSize(String address, String network) { + var index = 0; + List result = null; + do { + result = web3Functions.callFunction(new Function( + BALANCES, + List.of(new Uint256(index)), + List.of( + new TypeReference() { + } + )), + address, + resolveBlock(null), + network + ); + + if (result != null) { + index++; + } + + } while(result != null); + + return index; + } + public Optional getCurveTokenInfo(String minter, Long block, String network, long i) { var coin = web3Functions.callFunction(new Function( COINS, diff --git a/src/main/java/pro/belbix/ethparser/web3/prices/PriceOracle.java b/src/main/java/pro/belbix/ethparser/web3/prices/PriceOracle.java index 384f9a58..64bc2d6b 100644 --- a/src/main/java/pro/belbix/ethparser/web3/prices/PriceOracle.java +++ b/src/main/java/pro/belbix/ethparser/web3/prices/PriceOracle.java @@ -86,7 +86,7 @@ public String getLargestKeyToken(String tokenAddress, long block, String network } } - private String getOracleAddress(String tokenAddress, long block, String network) { + public String getOracleAddress(String tokenAddress, long block, String network) { if (BSC_NETWORK.equals(network)) { Optional factory = functionsUtils.callStrByName("factory", tokenAddress, block, network); diff --git a/src/main/java/pro/belbix/ethparser/web3/prices/PriceProvider.java b/src/main/java/pro/belbix/ethparser/web3/prices/PriceProvider.java index d4ca1a73..2b9fa8a2 100644 --- a/src/main/java/pro/belbix/ethparser/web3/prices/PriceProvider.java +++ b/src/main/java/pro/belbix/ethparser/web3/prices/PriceProvider.java @@ -5,7 +5,6 @@ import static pro.belbix.ethparser.web3.abi.FunctionsNames.MINTER; import static pro.belbix.ethparser.web3.abi.FunctionsNames.TOTAL_SUPPLY; import static pro.belbix.ethparser.web3.contracts.ContractConstants.EXCLUDE_JARVIS_STABLECOIN; -import static pro.belbix.ethparser.web3.contracts.ContractConstants.IS_NOT_AVAILABLE_IN_ORACLE; import static pro.belbix.ethparser.web3.contracts.ContractConstants.ZERO_ADDRESS; import static pro.belbix.ethparser.web3.contracts.ContractUtils.getBaseNetworkWrappedTokenAddress; @@ -116,7 +115,7 @@ public double getCurvePrice(String address, Long block, String network) { if (checkAddress.isEmpty()) { return getPriceForCoin(address, block, network); } - size = 2; + size = functionsUtils.getCurveVaultSize(address, network); minter = address; } @@ -167,7 +166,7 @@ public double getLpTokenUsdPriceFromEth(String lpAddress, double amount, long bl return 0.0; } - if (PriceOracle.isAvailable(block, network) && IS_NOT_AVAILABLE_IN_ORACLE.get(network).stream().noneMatch(i -> i.equalsIgnoreCase(lpAddress))) { + if (PriceOracle.isAvailable(block, network) && functionsUtils.canGetTokenPrice(lpAddress, priceOracle.getOracleAddress(lpAddress, block, network), block, network)) { return amount * priceOracle.getPriceForCoinOnChain(lpAddress, block, network); } log.info("Oracle not deployed yet, use direct calculation for prices"); @@ -193,8 +192,14 @@ private double calculateLpTokenPrice(String lpAddress, long block, String network ) { - Tuple2 tokensAdr = contractDbService - .tokenAddressesByUniPairAddress(lpAddress, network); + Tuple2 tokensAdr = null; + + try { + tokensAdr = contractDbService + .tokenAddressesByUniPairAddress(lpAddress, network); + } catch (IllegalStateException e) { + tokensAdr = functionsUtils.callTokensForSwapPlatform(lpAddress, network); + } double positionFraction = amount / lpBalance; @@ -276,7 +281,7 @@ private double getPriceForCoinFromEth(String address, Long block, String network return DEFAULT_RETURN_PRICE; } - if (PriceOracle.isAvailable(block, network)) { + if (PriceOracle.isAvailable(block, network) && functionsUtils.canGetTokenPrice(address, priceOracle.getOracleAddress(address, block, network), block, network)) { return priceOracle.getPriceForCoinOnChain(address, block, network); } log.debug("Oracle not deployed yet, use direct calculation for prices"); @@ -320,6 +325,11 @@ private double getPriceForCoinFromEthLegacy( if (curveUnderlying != null) { return getPriceForCoinFromEthLegacy(curveUnderlying, block, network, handled); } +// +// var vaultName = functionsUtils.getName(address, network); +// if (UniPairType.isCurve(vaultName)) { +// return getCurvePrice(address, block, network); +// } } catch (Exception ignore) { } log.error("Not found lp for {}, block: {}, network: {}", address, block, network); From 54d63a80d76c577088ae1aeea42a120c169eea3a Mon Sep 17 00:00:00 2001 From: alex Date: Mon, 28 Mar 2022 01:39:05 +0300 Subject: [PATCH 32/65] Add new unipair for bsc, fix problem with constant and calculate price for curve --- .../repositories/eth/VaultRepository.java | 5 ++ .../ethparser/service/SharePriceService.java | 4 +- .../service/external/CovalenthqService.java | 3 +- .../task/CovalenthqTransactionTask.java | 37 +++++++++++--- .../web3/contracts/ContractConstants.java | 3 +- .../web3/contracts/ContractConstantsV2.java | 33 ++++++++++++ .../web3/contracts/ContractConstantsV3.java | 50 +++++++++++++++++++ .../web3/contracts/ContractConstantsV4.java | 19 +++++++ .../web3/contracts/ContractConstantsV5.java | 16 ++++++ .../web3/contracts/ContractConstantsV6.java | 13 +++++ .../web3/contracts/ContractUtils.java | 17 +++---- .../ethparser/web3/contracts/UniPairType.java | 4 +- .../web3/erc20/db/TransferDBService.java | 11 ++-- .../ethparser/web3/prices/PriceProvider.java | 25 +++++++--- 14 files changed, 205 insertions(+), 35 deletions(-) create mode 100644 src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstantsV2.java create mode 100644 src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstantsV3.java create mode 100644 src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstantsV4.java create mode 100644 src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstantsV5.java create mode 100644 src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstantsV6.java diff --git a/src/main/java/pro/belbix/ethparser/repositories/eth/VaultRepository.java b/src/main/java/pro/belbix/ethparser/repositories/eth/VaultRepository.java index 2bf99ddf..61f8a4b3 100644 --- a/src/main/java/pro/belbix/ethparser/repositories/eth/VaultRepository.java +++ b/src/main/java/pro/belbix/ethparser/repositories/eth/VaultRepository.java @@ -34,6 +34,11 @@ VaultEntity findFirstByContract( + "where f1.network = :network") List findAllByNetwork(String network); + @Query("select t from VaultEntity t " + + "left join fetch t.contract f1 " + + "where f1.network = :network and f1.address in :address") + List findAllInContractAddress(List address, String network); + @Query("select max(t.id) from VaultEntity t") int findMaxId(); } diff --git a/src/main/java/pro/belbix/ethparser/service/SharePriceService.java b/src/main/java/pro/belbix/ethparser/service/SharePriceService.java index f624fa20..08db3d2e 100644 --- a/src/main/java/pro/belbix/ethparser/service/SharePriceService.java +++ b/src/main/java/pro/belbix/ethparser/service/SharePriceService.java @@ -13,7 +13,7 @@ import pro.belbix.ethparser.repositories.SharePriceRepository; import pro.belbix.ethparser.utils.ProfitUtils; import pro.belbix.ethparser.web3.abi.FunctionsUtils; -import pro.belbix.ethparser.web3.contracts.ContractConstants; +import pro.belbix.ethparser.web3.contracts.ContractConstantsV2; @Service @RequiredArgsConstructor @@ -32,7 +32,7 @@ public BigInteger getSharePrice(String vaultAddress, long block, String network) return sharePrice.get().getValue(); } - if (ContractConstants.EXCLUDE_ADDRESSES_FOR_PRICE_SHARE_BY_NETWORK.get(network).contains(vaultAddress.toLowerCase())) { + if (ContractConstantsV2.EXCLUDE_ADDRESSES_FOR_PRICE_SHARE_BY_NETWORK.get(network).contains(vaultAddress.toLowerCase())) { sharePriceRepository.save(new SharePrice(id, BigInteger.ONE)); return BigInteger.ONE; } diff --git a/src/main/java/pro/belbix/ethparser/service/external/CovalenthqService.java b/src/main/java/pro/belbix/ethparser/service/external/CovalenthqService.java index 79b94375..6d1d9b1c 100644 --- a/src/main/java/pro/belbix/ethparser/service/external/CovalenthqService.java +++ b/src/main/java/pro/belbix/ethparser/service/external/CovalenthqService.java @@ -43,10 +43,10 @@ public long getCreatedBlockByLastTransaction(String address, String network) { try { var page = 0; var result = getTransactionByAddress(address, network, true, false, page, TRANSFER_LIMIT); - if (result == null || result.getData() == null || result.getData().getItems() == null) { return 0; } + log.info("Got tx from covalenthq: {}", result.getData().getItems().size()); var createdTx = findCovalenthqTransactionItem(result); @@ -58,6 +58,7 @@ public long getCreatedBlockByLastTransaction(String address, String network) { || result.getData().getItems().isEmpty()) { return 0; } + log.info("Got tx from covalenthq: {}", result.getData().getItems().size()); createdTx = findCovalenthqTransactionItem(result); } diff --git a/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java b/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java index 28b96e25..2e000042 100644 --- a/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java +++ b/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java @@ -4,13 +4,14 @@ import static pro.belbix.ethparser.entity.profit.CovalenthqVaultTransactionType.DEPOSIT_UNI; import static pro.belbix.ethparser.entity.profit.CovalenthqVaultTransactionType.WITHDRAW; import static pro.belbix.ethparser.entity.profit.CovalenthqVaultTransactionType.WITHDRAW_UNI; -import static pro.belbix.ethparser.service.AbiProviderService.MATIC_NETWORK; +import static pro.belbix.ethparser.service.AbiProviderService.BSC_NETWORK; import java.math.BigDecimal; import java.math.BigInteger; import java.util.Collection; import java.util.HashSet; import java.util.LinkedList; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -48,8 +49,8 @@ public class CovalenthqTransactionTask { // max block range in covalenthq private static final int MAX_BLOCK_RANGE = 1000000; private static final int MINUS_BLOCK = 10; - private static final int PAGINATION_SIZE = 1000000; - private static final int PAGINATION_SIZE_FOR_A_LOT_OF_DATA = 1000; + private static final int PAGINATION_SIZE = 10000; + private static final int PAGINATION_SIZE_FOR_A_LOT_OF_DATA = 100; private static final Map LOG_EVENTS = Map.of( "0xf279e6a1f5e320cca91135676d9cb6e44ca8a08c0b88342bcdb1144f6511b568", WITHDRAW_UNI, "0x884edad9ce6fa2440d8a54cc123490eb96d2768479d49ff9c7366125a9424364", WITHDRAW, @@ -82,7 +83,30 @@ public void start() { log.info("Begin parse vault tx"); var executor = Executors.newFixedThreadPool(maxThreadSize); - vaultRepository.findAllByNetwork(MATIC_NETWORK).stream() + vaultRepository.findAllInContractAddress(List.of( + "0x1c4adff419f6b91e51d0ade953c9bbf5d16a583f", + "0xcd8fb1302c30fde56bce5b34211e84561bbf0df1", + "0xe3f309f151746b3c0953e4c0e455bff3dc2176aa", + "0x0a7d74604b39229d444855ef294f287099774ac8", + "0x129ccee12a9542ff77e066e6f8d7df49f8cbf89d", + "0x75071f2653fbc902ebaff908d4c68712a5d1c960", + "0x63671425ef4d25ec2b12c7d05de855c143f16e3b", + "0x14cb410659b4a4a7ccea99e6f6c9eac8718160df", + "0x5da237ad194b8bbb008ac8916df99a92a8a7c8eb", + "0xe64bfe13aa99335487f1f42a56cddbffaec83bbf", + "0xc97ddaa8091abaf79a4910b094830cce5cdd78f4", + "0x6d386490e2367fc31b4acc99ab7c7d4d998a3121", + "0xe1f9a3ee001a2ecc906e8de637dbf20bb2d44633", + "0x84646f736795a8bc22ab34e05c8982cd058328c7", + "0xe604fd5b1317babd0cf2c72f7f5f2ad8c00adbe1", + "0xffbd102fafbd9e15c9122d9c62ab299afd4d3e4f", + "0xf7a3a95d0f7e8a5eeae483cdd7b76af287283d34", + "0xf553e1f826f42716cdfe02bde5ee76b2a52fc7eb", + "0x6a0d7383762962be039c197462bf1df377410853", + "0x299b00d031ba65ca3a22a8f7e8059dab0b072247", + "0x299b00d031ba65ca3a22a8f7e8059dab0b072247" + ), + BSC_NETWORK).stream() .map(i -> CompletableFuture.runAsync(() -> getVaultTransaction(i), executor)) .collect(Collectors.toList()); @@ -94,7 +118,7 @@ private void getVaultTransaction(VaultEntity vault) { var contract = vault.getContract(); var lastTx = covalenthqVaultTransactionRepository.findAllByNetworkAndContractAddress(contract.getNetwork(), contract.getAddress(), PageRequest.of(0, 1, Sort.by("block").ascending())); - + log.info("Got response: {}", lastTx); var startingBlock = 0L; if (!lastTx.isEmpty()) { @@ -126,12 +150,13 @@ private void fetchTransactions(String address, String network, long startingBloc var transaction = covalenthqService.getTransactionByAddress(address, network, page, limit, startingBlock, endingBlock); var result = new LinkedList(transaction.getData().getItems()); var hasMore = transaction.getData().getPagination().isHasMore(); - + log.info("Covalnethq result: {}", result.size()); while (hasMore) { page++; transaction = covalenthqService.getTransactionByAddress(address, network, page, limit, startingBlock, endingBlock); hasMore = transaction.getData().getPagination().isHasMore(); result.addAll(transaction.getData().getItems()); + log.info("Covalnethq result: {}", result.size()); } var transactions = result.stream() diff --git a/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstants.java b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstants.java index a0a977c4..0ad35bed 100644 --- a/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstants.java +++ b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstants.java @@ -159,7 +159,8 @@ public class ContractConstants { "0x0000000000085d4780B73119b644AE5ecd22b376".toLowerCase(), //TUSD "0x2791bca1f2de4661ed88a30c99a7a9449aa84174".toLowerCase(), //matic USDC "0xc2132d05d31c914a87c6611c10748aeb04b58e8f".toLowerCase(), //matic USDT - "0xE840B73E5287865EEc17d250bFb1536704B43B21".toLowerCase() //matic mUSD + "0xE840B73E5287865EEc17d250bFb1536704B43B21".toLowerCase(), //matic mUSD + "0x2791Bca1f2de4661ED88A30C99A7a9449Aa84174".toLowerCase() //matic USDC PoS ); //Key tokens are used to find liquidity for any given token on Uni, Sushi and Curve. diff --git a/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstantsV2.java b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstantsV2.java new file mode 100644 index 00000000..1af8ecfd --- /dev/null +++ b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstantsV2.java @@ -0,0 +1,33 @@ +package pro.belbix.ethparser.web3.contracts; + +import static pro.belbix.ethparser.service.AbiProviderService.BSC_NETWORK; +import static pro.belbix.ethparser.service.AbiProviderService.ETH_NETWORK; +import static pro.belbix.ethparser.service.AbiProviderService.MATIC_NETWORK; + +import java.util.List; +import java.util.Map; + +public interface ContractConstantsV2 { + + Map> EXCLUDE_ADDRESSES_FOR_PRICE_SHARE_BY_NETWORK = Map.of( + ETH_NETWORK, List.of( + "0x25550Cccbd68533Fa04bFD3e3AC4D09f9e00Fc50".toLowerCase(), + "0x59258f4e15a5fc74a7284055a8094f58108dbd4f".toLowerCase() // POOL + ), + BSC_NETWORK, List.of(), + MATIC_NETWORK, List.of() + ); + + Map> FULL_PARSABLE_UNI_PAIRS = Map.of( + ETH_NETWORK, Map.of( + "0x514906fc121c7878424a5c928cad1852cc545892".toLowerCase(), 10777067, + // UNI_LP_USDC_FARM - FARM + "0x56feaccb7f750b997b36a68625c7c596f0b41a58".toLowerCase(), 11407437, + // UNI_LP_WETH_FARM - FARM + "0xb9fa44b0911f6d777faab2fa9d8ef103f25ddf49".toLowerCase(), 11407202 + // UNI_LP_GRAIN_FARM - GRAIN + ), + BSC_NETWORK, Map.of(), + MATIC_NETWORK, Map.of() + ); +} diff --git a/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstantsV3.java b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstantsV3.java new file mode 100644 index 00000000..2b8fe1ac --- /dev/null +++ b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstantsV3.java @@ -0,0 +1,50 @@ +package pro.belbix.ethparser.web3.contracts; + +import static pro.belbix.ethparser.service.AbiProviderService.BSC_NETWORK; +import static pro.belbix.ethparser.service.AbiProviderService.ETH_NETWORK; +import static pro.belbix.ethparser.service.AbiProviderService.MATIC_NETWORK; + +import java.util.List; +import java.util.Map; + +public interface ContractConstantsV3 { + + Map> PS_ADDRESSES_BY_NETWORK = Map.of( + ETH_NETWORK, List.of( + "0xd3093e3efbe00f010e8f5efe3f1cb5d9b7fe0eb1".toLowerCase(), // + "0x8f5adC58b32D4e5Ca02EAC0E293D35855999436C".toLowerCase(), // ST_PS + "0xa0246c9032bc3a600820415ae600c6388619a14d".toLowerCase(), // FARM TOKEN + "0x25550Cccbd68533Fa04bFD3e3AC4D09f9e00Fc50".toLowerCase(), // PS + "0x59258F4e15A5fC74A7284055A8094F58108dbD4f".toLowerCase() // PS_V0 + ), + BSC_NETWORK, List.of( + "0x4B5C23cac08a567ecf0c1fFcA8372A45a5D33743".toLowerCase() + ), + MATIC_NETWORK, List.of( + "0xab0b2ddb9c7e440fac8e140a89c0dbcbf2d7bbff".toLowerCase() + ) + ); + + Map> ORACLES = Map.of( + ETH_NETWORK, + Map.of(12015724L, "0x48DC32eCA58106f06b41dE514F29780FFA59c279".toLowerCase(), + 12820106L, "0x1358c91D5b25D3eDAc2b7B26A619163d78f1717d".toLowerCase()), + BSC_NETWORK, + Map.of(6442627L, "0xE0e9F05054Ad3a2b6414AD13D768be91a84b47e8".toLowerCase(), + 6952687L, "0x643cF46eef91Bd878D9710ceEB6a7E6F929F2608".toLowerCase(), + 9142012L, "0x0E74303d0D18884Ce2CEb3670e72686645c4f38B".toLowerCase()), + MATIC_NETWORK, + Map.of(16841617L, "0x0E74303d0D18884Ce2CEb3670e72686645c4f38B".toLowerCase()) + ); + + Map> ORACLES_BY_FACTORY = Map.of( + ETH_NETWORK, + Map.of("0x5C69bEe701ef814a2B6a3EDD4B1652CB9cc5aA6f".toLowerCase(), + "0x48DC32eCA58106f06b41dE514F29780FFA59c279".toLowerCase()), + BSC_NETWORK, + Map.of("0xbcfccbde45ce874adcb698cc183debcf17952812".toLowerCase(), + "0xE0e9F05054Ad3a2b6414AD13D768be91a84b47e8".toLowerCase(), // V1 + "0xca143ce32fe78f1f7019d7d551a6402fc5350c73".toLowerCase(), + "0x643cF46eef91Bd878D9710ceEB6a7E6F929F2608".toLowerCase()) // V2 + ); +} diff --git a/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstantsV4.java b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstantsV4.java new file mode 100644 index 00000000..c5ad4724 --- /dev/null +++ b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstantsV4.java @@ -0,0 +1,19 @@ +package pro.belbix.ethparser.web3.contracts; + +import static pro.belbix.ethparser.service.AbiProviderService.BSC_NETWORK; +import static pro.belbix.ethparser.service.AbiProviderService.ETH_NETWORK; +import static pro.belbix.ethparser.service.AbiProviderService.MATIC_NETWORK; + +import java.util.List; +import java.util.Map; + +public interface ContractConstantsV4 { + Map> EXCLUDE_JARVIS_STABLECOIN = Map.of( + ETH_NETWORK, List.of(), + BSC_NETWORK, List.of(), + MATIC_NETWORK, List.of( + "0x8ca194A3b22077359b5732DE53373D4afC11DeE3".toLowerCase(), // jCAD + "0x8343091F2499FD4b6174A46D067A920a3b851FF9".toLowerCase() // jJPY + ) + ); +} diff --git a/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstantsV5.java b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstantsV5.java new file mode 100644 index 00000000..2936a909 --- /dev/null +++ b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstantsV5.java @@ -0,0 +1,16 @@ +package pro.belbix.ethparser.web3.contracts; + +import java.util.List; + +public interface ContractConstantsV5 { + List ONE_DOLLAR_TOKENS = List.of( + "0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48".toLowerCase(), //USDC + "0xe9e7cea3dedca5984780bafc599bd69add087d56".toLowerCase(), //BUSD + "0xdAC17F958D2ee523a2206206994597C13D831ec7".toLowerCase(), //USDT + "0x0000000000085d4780B73119b644AE5ecd22b376".toLowerCase(), //TUSD + "0x2791bca1f2de4661ed88a30c99a7a9449aa84174".toLowerCase(), //matic USDC + "0xc2132d05d31c914a87c6611c10748aeb04b58e8f".toLowerCase(), //matic USDT + "0xE840B73E5287865EEc17d250bFb1536704B43B21".toLowerCase(), //matic mUSD + "0x2791Bca1f2de4661ED88A30C99A7a9449Aa84174".toLowerCase() //matic USDC PoS + ); +} diff --git a/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstantsV6.java b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstantsV6.java new file mode 100644 index 00000000..a1907ac3 --- /dev/null +++ b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstantsV6.java @@ -0,0 +1,13 @@ +package pro.belbix.ethparser.web3.contracts; + +import java.util.Set; + +public interface ContractConstantsV6 { + + Set PS_ADDRESSES = Set.of( + "0x8f5adC58b32D4e5Ca02EAC0E293D35855999436C".toLowerCase(), // ST_PS + "0xa0246c9032bc3a600820415ae600c6388619a14d".toLowerCase(), // FARM TOKEN + "0x25550Cccbd68533Fa04bFD3e3AC4D09f9e00Fc50".toLowerCase(), // PS + "0x59258F4e15A5fC74A7284055A8094F58108dbD4f".toLowerCase() // PS_V0 + ); +} diff --git a/src/main/java/pro/belbix/ethparser/web3/contracts/ContractUtils.java b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractUtils.java index d81fbdbc..ad3cd38d 100644 --- a/src/main/java/pro/belbix/ethparser/web3/contracts/ContractUtils.java +++ b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractUtils.java @@ -11,14 +11,9 @@ import static pro.belbix.ethparser.web3.contracts.ContractConstants.FULL_PARSABLE_UNI_PAIRS; import static pro.belbix.ethparser.web3.contracts.ContractConstants.MATIC_BLOCK_NUMBER_06_JUL_2021; import static pro.belbix.ethparser.web3.contracts.ContractConstants.MATIC_FARM_TOKEN; -import static pro.belbix.ethparser.web3.contracts.ContractConstants.ONE_DOLLAR_TOKENS; import static pro.belbix.ethparser.web3.contracts.ContractConstants.ONE_INCH_FACTORY_ADDRESS; import static pro.belbix.ethparser.web3.contracts.ContractConstants.ONE_INCH_FACTORY_BSC; -import static pro.belbix.ethparser.web3.contracts.ContractConstants.ORACLES; -import static pro.belbix.ethparser.web3.contracts.ContractConstants.ORACLES_BY_FACTORY; import static pro.belbix.ethparser.web3.contracts.ContractConstants.PARSABLE_BANCOR_TRANSACTIONS; -import static pro.belbix.ethparser.web3.contracts.ContractConstants.PS_ADDRESSES; -import static pro.belbix.ethparser.web3.contracts.ContractConstants.PS_ADDRESSES_BY_NETWORK; import static pro.belbix.ethparser.web3.contracts.ContractConstants.PS_V0_ADDRESS; import static pro.belbix.ethparser.web3.contracts.ContractConstants.ZERO_ADDRESS; @@ -52,14 +47,14 @@ public static boolean isPsAddress(String address) { if (address == null) { return false; } - return PS_ADDRESSES.contains(address.toLowerCase()); + return ContractConstantsV6.PS_ADDRESSES.contains(address.toLowerCase()); } public static boolean isPsAddress(String address, String network) { if (address == null) { return false; } - return PS_ADDRESSES_BY_NETWORK.get(network).contains(address.toLowerCase()); + return ContractConstantsV3.PS_ADDRESSES_BY_NETWORK.get(network).contains(address.toLowerCase()); } public static boolean isFarmAddress(String address) { @@ -69,7 +64,7 @@ public static boolean isFarmAddress(String address) { } public static boolean isStableCoin(String address) { - return ONE_DOLLAR_TOKENS.contains(address.toLowerCase()); + return ContractConstantsV5.ONE_DOLLAR_TOKENS.contains(address.toLowerCase()); } public static String getBaseAddressInsteadOfZero(String address, String network) { @@ -139,7 +134,7 @@ public static String getControllerAddressByBlockAndNetwork(long block, String ne } public static String getPriceOracle(long block, String network) { - Entry entry = new TreeMap<>(ORACLES.get(network)).floorEntry(block); + Entry entry = new TreeMap<>(ContractConstantsV3.ORACLES.get(network)).floorEntry(block); if (entry == null) { return null; } @@ -147,7 +142,7 @@ public static String getPriceOracle(long block, String network) { } public static String getPriceOracleByFactory(String factory, String network) { - String oracle = ORACLES_BY_FACTORY.get(network).get(factory); + String oracle = ContractConstantsV3.ORACLES_BY_FACTORY.get(network).get(factory); if (oracle == null) { throw new IllegalStateException("Factory " + factory + " not found"); } @@ -216,7 +211,7 @@ public static String getFarmAddress(String network) { } public static boolean isFullParsableLp(String address, String network) { - return FULL_PARSABLE_UNI_PAIRS.get(network).containsKey(address.toLowerCase()); + return ContractConstantsV2.FULL_PARSABLE_UNI_PAIRS.get(network).containsKey(address.toLowerCase()); } public static boolean isFullParsableLpAddressAndDate(String address, int date, diff --git a/src/main/java/pro/belbix/ethparser/web3/contracts/UniPairType.java b/src/main/java/pro/belbix/ethparser/web3/contracts/UniPairType.java index b10177f6..82aeeaaf 100644 --- a/src/main/java/pro/belbix/ethparser/web3/contracts/UniPairType.java +++ b/src/main/java/pro/belbix/ethparser/web3/contracts/UniPairType.java @@ -20,6 +20,8 @@ public enum UniPairType { PANCACKE_SWAP("PancakeSwap"), // Kyber KYBER("Kyber"), + // Ellipsis.finance .. + ELLIPSIS_FINANCE("Ellipsis.finance"), // Uniswap V2 UNISWAP("Uniswap"); @@ -41,6 +43,6 @@ public static boolean isBalancer(String name) { } public static boolean isCurve(String name) { - return Stream.of(CURVE).anyMatch(i -> name.toLowerCase().startsWith(i.getName())); + return Stream.of(CURVE, ELLIPSIS_FINANCE).anyMatch(i -> name.toLowerCase().startsWith(i.getName())); } } diff --git a/src/main/java/pro/belbix/ethparser/web3/erc20/db/TransferDBService.java b/src/main/java/pro/belbix/ethparser/web3/erc20/db/TransferDBService.java index 49410eb7..a30c6525 100644 --- a/src/main/java/pro/belbix/ethparser/web3/erc20/db/TransferDBService.java +++ b/src/main/java/pro/belbix/ethparser/web3/erc20/db/TransferDBService.java @@ -1,8 +1,5 @@ package pro.belbix.ethparser.web3.erc20.db; -import static pro.belbix.ethparser.web3.contracts.ContractConstants.PS_ADDRESS; -import static pro.belbix.ethparser.web3.contracts.ContractConstants.PS_V0_ADDRESS; -import static pro.belbix.ethparser.web3.contracts.ContractConstants.ZERO_ADDRESS; import static pro.belbix.ethparser.web3.erc20.TransferType.KEEP_OWNERSHIP; import static pro.belbix.ethparser.web3.erc20.TransferType.LP_BUY; import static pro.belbix.ethparser.web3.erc20.TransferType.LP_SELL; @@ -32,10 +29,10 @@ public class TransferDBService { private static final Set notCheckableAddresses = new HashSet<>(); static { - notCheckableAddresses.add("0x8f5adC58b32D4e5Ca02EAC0E293D35855999436C"); // st_ps - notCheckableAddresses.add(PS_ADDRESS); //ps - notCheckableAddresses.add(PS_V0_ADDRESS); // ps_v0 - notCheckableAddresses.add(ZERO_ADDRESS); + notCheckableAddresses.add("0x8f5adC58b32D4e5Ca02EAC0E293D35855999436C".toLowerCase()); // st_ps + notCheckableAddresses.add("0x25550Cccbd68533Fa04bFD3e3AC4D09f9e00Fc50".toLowerCase()); //ps + notCheckableAddresses.add("0x59258F4e15A5fC74A7284055A8094F58108dbD4f".toLowerCase()); // ps_v0 + notCheckableAddresses.add("0x0000000000000000000000000000000000000000".toLowerCase()); } private final Pageable limitOne = PageRequest.of(0, 1); diff --git a/src/main/java/pro/belbix/ethparser/web3/prices/PriceProvider.java b/src/main/java/pro/belbix/ethparser/web3/prices/PriceProvider.java index 2b9fa8a2..757cfaef 100644 --- a/src/main/java/pro/belbix/ethparser/web3/prices/PriceProvider.java +++ b/src/main/java/pro/belbix/ethparser/web3/prices/PriceProvider.java @@ -3,8 +3,8 @@ import static java.util.Objects.requireNonNullElse; import static pro.belbix.ethparser.web3.abi.FunctionsNames.GET_VAULT; import static pro.belbix.ethparser.web3.abi.FunctionsNames.MINTER; +import static pro.belbix.ethparser.web3.abi.FunctionsNames.NAME; import static pro.belbix.ethparser.web3.abi.FunctionsNames.TOTAL_SUPPLY; -import static pro.belbix.ethparser.web3.contracts.ContractConstants.EXCLUDE_JARVIS_STABLECOIN; import static pro.belbix.ethparser.web3.contracts.ContractConstants.ZERO_ADDRESS; import static pro.belbix.ethparser.web3.contracts.ContractUtils.getBaseNetworkWrappedTokenAddress; @@ -16,6 +16,7 @@ import java.util.Set; import java.util.TreeMap; import lombok.extern.log4j.Log4j2; +import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Service; import org.web3j.tuples.generated.Tuple2; import pro.belbix.ethparser.entity.contracts.ContractEntity; @@ -24,8 +25,10 @@ import pro.belbix.ethparser.repositories.v0.PriceRepository; import pro.belbix.ethparser.web3.abi.FunctionsNames; import pro.belbix.ethparser.web3.abi.FunctionsUtils; +import pro.belbix.ethparser.web3.contracts.ContractConstantsV4; import pro.belbix.ethparser.web3.contracts.ContractType; import pro.belbix.ethparser.web3.contracts.ContractUtils; +import pro.belbix.ethparser.web3.contracts.UniPairType; import pro.belbix.ethparser.web3.contracts.db.ContractDbService; @Service @@ -106,7 +109,6 @@ public double getBalancerPrice(String address, Long block, String network) { // TODO 0xc27bfe32e0a934a12681c1b35acf0dba0e7460ba has 0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee coin address public double getCurvePrice(String address, Long block, String network) { - var size = DEFAULT_CURVE_SIZE; var minter = functionsUtils.callAddressByName(MINTER, address, block, network) .orElse(null); @@ -115,10 +117,10 @@ public double getCurvePrice(String address, Long block, String network) { if (checkAddress.isEmpty()) { return getPriceForCoin(address, block, network); } - size = functionsUtils.getCurveVaultSize(address, network); minter = address; } + var size = functionsUtils.getCurveVaultSize(minter, network);; var decimal = functionsUtils.callIntByName(FunctionsNames.DECIMALS, address, block, network) .orElseThrow(() -> { log.error("Can not get decimal for {} {}", address, network); @@ -146,7 +148,7 @@ public double getCurvePrice(String address, Long block, String network) { throw new CanNotFetchPriceException(); }).intValue(); - var tokenPrice = getPriceForCoin(tokenInfo.getAddress(), block, network); + var tokenPrice = getPriceForCoinOrCurve(tokenInfo.getAddress(), block, network); if (tokenPrice == 0) { log.error("Can not fetch price for curve {} {}", address, network); @@ -169,6 +171,7 @@ public double getLpTokenUsdPriceFromEth(String lpAddress, double amount, long bl if (PriceOracle.isAvailable(block, network) && functionsUtils.canGetTokenPrice(lpAddress, priceOracle.getOracleAddress(lpAddress, block, network), block, network)) { return amount * priceOracle.getPriceForCoinOnChain(lpAddress, block, network); } + log.info("Oracle not deployed yet, use direct calculation for prices"); Tuple2 lpPooled = functionsUtils.callReserves( lpAddress, block, network); @@ -277,11 +280,11 @@ private double getPriceForCoinFromEth(String address, Long block, String network return 0.0; } - if (EXCLUDE_JARVIS_STABLECOIN.get(network).stream().anyMatch(i -> i.equals(address.toLowerCase()))) { + if (ContractConstantsV4.EXCLUDE_JARVIS_STABLECOIN.get(network).stream().anyMatch(i -> i.equals(address.toLowerCase()))) { return DEFAULT_RETURN_PRICE; } - if (PriceOracle.isAvailable(block, network) && functionsUtils.canGetTokenPrice(address, priceOracle.getOracleAddress(address, block, network), block, network)) { + if (PriceOracle.isAvailable(block, network)) { return priceOracle.getPriceForCoinOnChain(address, block, network); } log.debug("Oracle not deployed yet, use direct calculation for prices"); @@ -423,4 +426,14 @@ public boolean isDivisionSequenceSecondDividesFirst( private double normalizePrecision(Double amount, int decimal) { return amount * DEFAULT_POW.pow(DEFAULT_DECIMAL).doubleValue() / DEFAULT_POW.pow(decimal).doubleValue(); } + + private Double getPriceForCoinOrCurve(String address, Long block, String network) { + var name = functionsUtils.callStrByName(NAME, address, block, network) + .orElse(StringUtils.EMPTY); + if (UniPairType.isCurve(name)) { + return getCurvePrice(address, block, network); + } + + return getPriceForCoin(address, block, network); + } } From c1ebb32bba7302d5b3999f06341826f4ab1d3d08 Mon Sep 17 00:00:00 2001 From: alex Date: Mon, 4 Apr 2022 10:58:21 +0300 Subject: [PATCH 33/65] Fix problem with calculate price Denarius and Aureus --- .../ethparser/model/AbiProviderResponse.java | 13 +++++ .../pro/belbix/ethparser/model/TokenInfo.java | 20 ++++++++ .../ethparser/service/AbiProviderService.java | 26 ++++++++++ .../task/CovalenthqTransactionTask.java | 31 +++--------- .../ethparser/web3/EthBlockService.java | 19 +++++--- .../web3/contracts/ContractConstantsV7.java | 48 +++++++++++++++++++ .../web3/contracts/ContractUtils.java | 8 ++++ .../ethparser/web3/prices/PriceOracle.java | 10 ++++ .../ethparser/web3/prices/PriceProvider.java | 26 ++++++---- .../service/AbiProviderServiceTest.java | 26 ++++++++++ 10 files changed, 188 insertions(+), 39 deletions(-) create mode 100644 src/main/java/pro/belbix/ethparser/model/AbiProviderResponse.java create mode 100644 src/main/java/pro/belbix/ethparser/model/TokenInfo.java create mode 100644 src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstantsV7.java diff --git a/src/main/java/pro/belbix/ethparser/model/AbiProviderResponse.java b/src/main/java/pro/belbix/ethparser/model/AbiProviderResponse.java new file mode 100644 index 00000000..fba799c4 --- /dev/null +++ b/src/main/java/pro/belbix/ethparser/model/AbiProviderResponse.java @@ -0,0 +1,13 @@ +package pro.belbix.ethparser.model; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import lombok.AccessLevel; +import lombok.Data; +import lombok.experimental.FieldDefaults; + +@Data +@FieldDefaults(level = AccessLevel.PRIVATE) +@JsonIgnoreProperties(ignoreUnknown = true) +public class AbiProviderResponse { + long result; +} diff --git a/src/main/java/pro/belbix/ethparser/model/TokenInfo.java b/src/main/java/pro/belbix/ethparser/model/TokenInfo.java new file mode 100644 index 00000000..06b32523 --- /dev/null +++ b/src/main/java/pro/belbix/ethparser/model/TokenInfo.java @@ -0,0 +1,20 @@ +package pro.belbix.ethparser.model; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import lombok.experimental.FieldDefaults; + +@Data +@NoArgsConstructor +@AllArgsConstructor +@Builder +@FieldDefaults(level = AccessLevel.PRIVATE) +@EqualsAndHashCode +public class TokenInfo { + String address; + String network; +} diff --git a/src/main/java/pro/belbix/ethparser/service/AbiProviderService.java b/src/main/java/pro/belbix/ethparser/service/AbiProviderService.java index c25246f6..62eb4dd9 100644 --- a/src/main/java/pro/belbix/ethparser/service/AbiProviderService.java +++ b/src/main/java/pro/belbix/ethparser/service/AbiProviderService.java @@ -11,13 +11,17 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Optional; import lombok.Data; import lombok.extern.log4j.Log4j2; import org.springframework.http.ResponseEntity; import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter; +import org.springframework.stereotype.Service; import org.springframework.web.client.RestTemplate; +import pro.belbix.ethparser.model.AbiProviderResponse; @Log4j2 +@Service public class AbiProviderService { public final static int RATE_TIMEOUT = 1000; @@ -29,6 +33,7 @@ public class AbiProviderService { private final static String BSC_URL = "https://api.bscscan.com/api"; private final static String MATIC_URL = "https://api.polygonscan.com/api"; private final static String PARAMS = "?module={module}&action={action}&address={address}&apikey={apikey}"; + private final static String PARAMS_FOR_BLOCK = "?module={module}&action={action}&apikey={apikey}&closest={closest}×tamp={timestamp}"; private final RestTemplate restTemplate = new RestTemplate(); private Instant lastRequest = Instant.now(); @@ -92,6 +97,27 @@ public SourceCodeResult contractSourceCode(String address, String apiKey, String return result; } + public long getBlockByTimestamp(String timestamp, String network, String apiKey) { + var params = Map.of( + "module", "block", + "action", "getblocknobytime", + "apikey", apiKey, + "closest", "after", + "timestamp", timestamp + ); + + var result = restTemplate.getForObject( + getUrl(network) + PARAMS_FOR_BLOCK, + AbiProviderResponse.class, + params + ); + + return Optional.ofNullable(result) + .orElse(new AbiProviderResponse()) + .getResult(); + + } + private ResponseEntity sendRequest(Map vars, String network) { int count = 0; while (true) { diff --git a/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java b/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java index 2e000042..08d53bd8 100644 --- a/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java +++ b/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java @@ -4,7 +4,7 @@ import static pro.belbix.ethparser.entity.profit.CovalenthqVaultTransactionType.DEPOSIT_UNI; import static pro.belbix.ethparser.entity.profit.CovalenthqVaultTransactionType.WITHDRAW; import static pro.belbix.ethparser.entity.profit.CovalenthqVaultTransactionType.WITHDRAW_UNI; -import static pro.belbix.ethparser.service.AbiProviderService.BSC_NETWORK; +import static pro.belbix.ethparser.service.AbiProviderService.MATIC_NETWORK; import java.math.BigDecimal; import java.math.BigInteger; @@ -84,29 +84,12 @@ public void start() { var executor = Executors.newFixedThreadPool(maxThreadSize); vaultRepository.findAllInContractAddress(List.of( - "0x1c4adff419f6b91e51d0ade953c9bbf5d16a583f", - "0xcd8fb1302c30fde56bce5b34211e84561bbf0df1", - "0xe3f309f151746b3c0953e4c0e455bff3dc2176aa", - "0x0a7d74604b39229d444855ef294f287099774ac8", - "0x129ccee12a9542ff77e066e6f8d7df49f8cbf89d", - "0x75071f2653fbc902ebaff908d4c68712a5d1c960", - "0x63671425ef4d25ec2b12c7d05de855c143f16e3b", - "0x14cb410659b4a4a7ccea99e6f6c9eac8718160df", - "0x5da237ad194b8bbb008ac8916df99a92a8a7c8eb", - "0xe64bfe13aa99335487f1f42a56cddbffaec83bbf", - "0xc97ddaa8091abaf79a4910b094830cce5cdd78f4", - "0x6d386490e2367fc31b4acc99ab7c7d4d998a3121", - "0xe1f9a3ee001a2ecc906e8de637dbf20bb2d44633", - "0x84646f736795a8bc22ab34e05c8982cd058328c7", - "0xe604fd5b1317babd0cf2c72f7f5f2ad8c00adbe1", - "0xffbd102fafbd9e15c9122d9c62ab299afd4d3e4f", - "0xf7a3a95d0f7e8a5eeae483cdd7b76af287283d34", - "0xf553e1f826f42716cdfe02bde5ee76b2a52fc7eb", - "0x6a0d7383762962be039c197462bf1df377410853", - "0x299b00d031ba65ca3a22a8f7e8059dab0b072247", - "0x299b00d031ba65ca3a22a8f7e8059dab0b072247" - ), - BSC_NETWORK).stream() + "0x4A5eb54018462803B9D9a35D1730e45D6EE7033E", + "0x8cccdEBF657F072D83B2d94068C4377a3BA91e08", + "0xFf127E0dDB5E0a53d29f9eA029F5b41F281F3EFD", + "0x102Df50dB22407B64a8A6b11734c8743B6AeF953" + ), + MATIC_NETWORK).stream() .map(i -> CompletableFuture.runAsync(() -> getVaultTransaction(i), executor)) .collect(Collectors.toList()); diff --git a/src/main/java/pro/belbix/ethparser/web3/EthBlockService.java b/src/main/java/pro/belbix/ethparser/web3/EthBlockService.java index dbeb3138..a6445f08 100644 --- a/src/main/java/pro/belbix/ethparser/web3/EthBlockService.java +++ b/src/main/java/pro/belbix/ethparser/web3/EthBlockService.java @@ -1,23 +1,24 @@ package pro.belbix.ethparser.web3; +import lombok.AllArgsConstructor; import lombok.extern.log4j.Log4j2; +import org.springframework.cache.annotation.Cacheable; import org.springframework.stereotype.Service; import org.web3j.protocol.core.methods.response.EthBlock.Block; import pro.belbix.ethparser.entity.v0.BlockCacheEntity; +import pro.belbix.ethparser.properties.NetworkProperties; import pro.belbix.ethparser.repositories.v0.BlockCacheRepository; +import pro.belbix.ethparser.service.AbiProviderService; @Service @Log4j2 +@AllArgsConstructor public class EthBlockService { private final Web3Functions web3; private final BlockCacheRepository blockCacheRepository; - - public EthBlockService(Web3Functions web3, - BlockCacheRepository blockCacheRepository) { - this.web3 = web3; - this.blockCacheRepository = blockCacheRepository; - } + private final NetworkProperties networkProperties; + private final AbiProviderService abiProviderService; public synchronized long getTimestampSecForBlock(long blockNumber, String network) { BlockCacheEntity cachedBlock = @@ -44,6 +45,12 @@ public synchronized long getTimestampSecForBlock(long blockNumber, String networ return extractDateFromBlock(block); } + @Cacheable("timestamp_block") + public long getBlockFromOtherChain(long block, String networkFrom, String networkTo) { + var timestamp = getTimestampSecForBlock(block, networkFrom); + return abiProviderService.getBlockByTimestamp(String.valueOf(timestamp), networkTo, networkProperties.get(networkTo).getAbiProviderKey()); + } + private static long extractDateFromBlock(Block block) { return block.getTimestamp().longValue(); } diff --git a/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstantsV7.java b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstantsV7.java new file mode 100644 index 00000000..6cec2f48 --- /dev/null +++ b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstantsV7.java @@ -0,0 +1,48 @@ +package pro.belbix.ethparser.web3.contracts; + +import static pro.belbix.ethparser.service.AbiProviderService.BSC_NETWORK; +import static pro.belbix.ethparser.service.AbiProviderService.MATIC_NETWORK; + +import java.util.Map; +import pro.belbix.ethparser.model.TokenInfo; + +public interface ContractConstantsV7 { + Map COIN_PRICE_IN_OTHER_CHAIN = Map.of( + // Denarius (DEN-0121) to Denarius BSC + TokenInfo.builder() + .address("0xf379CB529aE58E1A03E62d3e31565f4f7c1F2020".toLowerCase()) + .network(MATIC_NETWORK) + .build(), + TokenInfo.builder() + .address("0xf6B53b4c982b9B7e87af9dc5c66C85117A5df303") + .network(BSC_NETWORK) + .build(), + // Denarius (DEN-MAR22) to Denarius BSC + TokenInfo.builder() + .address("0xa286eeDAa5aBbAE98F65b152B5057b8bE9893fbB".toLowerCase()) + .network(MATIC_NETWORK) + .build(), + TokenInfo.builder() + .address("0xf6B53b4c982b9B7e87af9dc5c66C85117A5df303") + .network(BSC_NETWORK) + .build(), + // Aureus (AUR-FEB22) to Aureus + TokenInfo.builder() + .address("0x6Fb2415463e949aF08ce50F83E94b7e008BABf07".toLowerCase()) + .network(MATIC_NETWORK) + .build(), + TokenInfo.builder() + .address("0xe8F6311A615b4E5f50bb2C6071c725518207337d") + .network(BSC_NETWORK) + .build(), + // Aureus (AUR-APR22) to Aureus + TokenInfo.builder() + .address("0xBF06D9b11126B140788D842a6ed8dC7885C722B3".toLowerCase()) + .network(MATIC_NETWORK) + .build(), + TokenInfo.builder() + .address("0xe8F6311A615b4E5f50bb2C6071c725518207337d") + .network(BSC_NETWORK) + .build() + ); +} diff --git a/src/main/java/pro/belbix/ethparser/web3/contracts/ContractUtils.java b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractUtils.java index ad3cd38d..e945e9dc 100644 --- a/src/main/java/pro/belbix/ethparser/web3/contracts/ContractUtils.java +++ b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractUtils.java @@ -141,6 +141,14 @@ public static String getPriceOracle(long block, String network) { return entry.getValue(); } + public static String getOldPriceOracle(String network) { + Entry entry = new TreeMap<>(ContractConstantsV3.ORACLES.get(network)).firstEntry(); + if (entry == null) { + return null; + } + return entry.getValue(); + } + public static String getPriceOracleByFactory(String factory, String network) { String oracle = ContractConstantsV3.ORACLES_BY_FACTORY.get(network).get(factory); if (oracle == null) { diff --git a/src/main/java/pro/belbix/ethparser/web3/prices/PriceOracle.java b/src/main/java/pro/belbix/ethparser/web3/prices/PriceOracle.java index 64bc2d6b..2f6a130e 100644 --- a/src/main/java/pro/belbix/ethparser/web3/prices/PriceOracle.java +++ b/src/main/java/pro/belbix/ethparser/web3/prices/PriceOracle.java @@ -48,6 +48,16 @@ public double getPriceForCoinOnChain(String tokenAdr, Long block, String network "Can't fetch price for " + tokenAdr)) .doubleValue(); + if (price == 0 && BSC_NETWORK.equals(network)) { + log.error("Can not get price by oracle address - {}, try use old oracle", oracleAddress); + oracleAddress = ContractUtils.getOldPriceOracle(network); + price = functionsUtils + .callIntByNameWithAddressArg(GET_PRICE, tokenAdr, oracleAddress, block, network) + .orElseThrow(() -> new IllegalStateException( + "Can't fetch price for " + tokenAdr)) + .doubleValue(); + } + return price / D18; } diff --git a/src/main/java/pro/belbix/ethparser/web3/prices/PriceProvider.java b/src/main/java/pro/belbix/ethparser/web3/prices/PriceProvider.java index 757cfaef..4468d213 100644 --- a/src/main/java/pro/belbix/ethparser/web3/prices/PriceProvider.java +++ b/src/main/java/pro/belbix/ethparser/web3/prices/PriceProvider.java @@ -15,17 +15,20 @@ import java.util.Map.Entry; import java.util.Set; import java.util.TreeMap; +import lombok.AllArgsConstructor; import lombok.extern.log4j.Log4j2; import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Service; import org.web3j.tuples.generated.Tuple2; import pro.belbix.ethparser.entity.contracts.ContractEntity; import pro.belbix.ethparser.error.exceptions.CanNotFetchPriceException; +import pro.belbix.ethparser.model.TokenInfo; import pro.belbix.ethparser.properties.AppProperties; -import pro.belbix.ethparser.repositories.v0.PriceRepository; +import pro.belbix.ethparser.web3.EthBlockService; import pro.belbix.ethparser.web3.abi.FunctionsNames; import pro.belbix.ethparser.web3.abi.FunctionsUtils; import pro.belbix.ethparser.web3.contracts.ContractConstantsV4; +import pro.belbix.ethparser.web3.contracts.ContractConstantsV7; import pro.belbix.ethparser.web3.contracts.ContractType; import pro.belbix.ethparser.web3.contracts.ContractUtils; import pro.belbix.ethparser.web3.contracts.UniPairType; @@ -33,6 +36,7 @@ @Service @Log4j2 +@AllArgsConstructor public class PriceProvider { private final static Double DEFAULT_RETURN_PRICE = 1D; @@ -46,15 +50,8 @@ public class PriceProvider { private final AppProperties appProperties; private final PriceOracle priceOracle; private final ContractDbService contractDbService; + private final EthBlockService ethBlockService; - public PriceProvider(FunctionsUtils functionsUtils, PriceRepository priceRepository, - AppProperties appProperties, PriceOracle priceOracle, - ContractDbService contractDbService) { - this.functionsUtils = functionsUtils; - this.appProperties = appProperties; - this.priceOracle = priceOracle; - this.contractDbService = contractDbService; - } public double getLpTokenUsdPrice(String lpAddress, double amount, long block, String network) { return getLpTokenUsdPriceFromEth(lpAddress, amount, block, network); @@ -284,6 +281,17 @@ private double getPriceForCoinFromEth(String address, Long block, String network return DEFAULT_RETURN_PRICE; } + final TokenInfo tokenInfo = TokenInfo.builder() + .address(address) + .network(network) + .build(); + + if (ContractConstantsV7.COIN_PRICE_IN_OTHER_CHAIN.containsKey(tokenInfo)) { + var tokenInfoInOtherChain = ContractConstantsV7.COIN_PRICE_IN_OTHER_CHAIN.get(tokenInfo); + var otherBlockChain = ethBlockService.getBlockFromOtherChain(block, network, tokenInfoInOtherChain.getNetwork()); + return priceOracle.getPriceForCoinOnChain(tokenInfoInOtherChain.getAddress(), otherBlockChain, tokenInfoInOtherChain.getNetwork()); + } + if (PriceOracle.isAvailable(block, network)) { return priceOracle.getPriceForCoinOnChain(address, block, network); } diff --git a/src/test/java/pro/belbix/ethparser/service/AbiProviderServiceTest.java b/src/test/java/pro/belbix/ethparser/service/AbiProviderServiceTest.java index 6b408e81..124eb5b8 100644 --- a/src/test/java/pro/belbix/ethparser/service/AbiProviderServiceTest.java +++ b/src/test/java/pro/belbix/ethparser/service/AbiProviderServiceTest.java @@ -1,16 +1,42 @@ package pro.belbix.ethparser.service; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static pro.belbix.ethparser.service.AbiProviderService.BSC_NETWORK; import static pro.belbix.ethparser.service.AbiProviderService.ETH_NETWORK; +import static pro.belbix.ethparser.service.AbiProviderService.MATIC_NETWORK; import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.ContextConfiguration; +import pro.belbix.ethparser.Application; +import pro.belbix.ethparser.properties.NetworkProperties; +import pro.belbix.ethparser.web3.EthBlockService; +@SpringBootTest(classes = Application.class) +@ContextConfiguration class AbiProviderServiceTest { + @Autowired + EthBlockService ethBlockService; + @Autowired + AbiProviderService abiProviderService; + @Autowired + NetworkProperties networkProperties; + @Test void contractSourceCode() { AbiProviderService abiProviderService = new AbiProviderService(); assertNotNull(abiProviderService.contractSourceCode( "0xBB9bc244D798123fDe783fCc1C72d3Bb8C189413", "YourApiKeyToken", ETH_NETWORK)); } + + @Test + void getBlockInDifferentChainAtTheSameTimeTest() { + var timestamp = ethBlockService.getTimestampSecForBlock(25561696, MATIC_NETWORK); + var result = abiProviderService.getBlockByTimestamp(String.valueOf(timestamp), BSC_NETWORK, networkProperties.get(BSC_NETWORK).getAbiProviderKey()); + + assertEquals(15758669L, result); + } } From 4a3c96ac730aeecb7da5fb695b22fbe384284026 Mon Sep 17 00:00:00 2001 From: alex Date: Mon, 4 Apr 2022 11:04:41 +0300 Subject: [PATCH 34/65] Fix problem with NaN --- .../java/pro/belbix/ethparser/service/TokenPriceService.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/pro/belbix/ethparser/service/TokenPriceService.java b/src/main/java/pro/belbix/ethparser/service/TokenPriceService.java index ef06c49b..4cc93548 100644 --- a/src/main/java/pro/belbix/ethparser/service/TokenPriceService.java +++ b/src/main/java/pro/belbix/ethparser/service/TokenPriceService.java @@ -59,7 +59,10 @@ public Double getTokenPrice(String vaultAddress, BigDecimal amount, long block, if (UniPairType.isLpUniPair(name)) { // priceProvider.getLpTokenUsdPrice return price * amount - price = priceProvider.getLpTokenUsdPrice(underlyingAddress, amount.doubleValue(), block, network) / amount.doubleValue(); + price = priceProvider.getLpTokenUsdPrice(underlyingAddress, amount.doubleValue(), block, network); + if (!amount.equals(BigDecimal.ZERO)) { + price = price / amount.doubleValue(); + } } else if (UniPairType.isBalancer(name)) { price = priceProvider.getBalancerPrice(underlyingAddress, block, network); log.info("Fetched price balancer: {} {} = {}", underlyingAddress, network, price); From 1c06fba7f6029d329b753e14188a36dd3ca8d2c6 Mon Sep 17 00:00:00 2001 From: alex Date: Tue, 5 Apr 2022 02:53:40 +0300 Subject: [PATCH 35/65] Added support for UniSwapV3 --- .../ethparser/service/TokenPriceService.java | 8 ++ .../ethparser/web3/abi/FunctionsNames.java | 1 + .../ethparser/web3/abi/FunctionsUtils.java | 18 ++- .../web3/contracts/UniswapV3Pools.java | 129 ++++++++++++++++++ .../ethparser/web3/prices/PriceProvider.java | 32 +++++ .../ethparser/web3/PriceProviderTest.java | 12 ++ 6 files changed, 199 insertions(+), 1 deletion(-) create mode 100644 src/main/java/pro/belbix/ethparser/web3/contracts/UniswapV3Pools.java diff --git a/src/main/java/pro/belbix/ethparser/service/TokenPriceService.java b/src/main/java/pro/belbix/ethparser/service/TokenPriceService.java index 4cc93548..7d2db1a3 100644 --- a/src/main/java/pro/belbix/ethparser/service/TokenPriceService.java +++ b/src/main/java/pro/belbix/ethparser/service/TokenPriceService.java @@ -18,6 +18,7 @@ import pro.belbix.ethparser.web3.abi.FunctionsUtils; import pro.belbix.ethparser.web3.contracts.ContractUtils; import pro.belbix.ethparser.web3.contracts.UniPairType; +import pro.belbix.ethparser.web3.contracts.UniswapV3Pools; import pro.belbix.ethparser.web3.prices.PriceProvider; @Service @@ -48,6 +49,13 @@ public Double getTokenPrice(String vaultAddress, BigDecimal amount, long block, return priceProvider.getPriceForCoin(ContractUtils.getFarmAddress(network), block, network); } + // UniSwapV3 + if (UniswapV3Pools.VAULTS_TO_POOLS.get(network).containsKey(vaultAddress.toLowerCase())) { + price = priceProvider.getUniswapV3Price(vaultAddress, block, network); + tokenPriceRepository.save(new TokenPrice(id, price)); + return price; + } + var underlyingAddress = functionsUtils.callAddressByName(UNDERLYING, vaultAddress, block, network) .orElseThrow(() -> { log.error("Can not fetch underlying address for {} {}", vaultAddress, network); diff --git a/src/main/java/pro/belbix/ethparser/web3/abi/FunctionsNames.java b/src/main/java/pro/belbix/ethparser/web3/abi/FunctionsNames.java index 7d7e430d..8788d70c 100644 --- a/src/main/java/pro/belbix/ethparser/web3/abi/FunctionsNames.java +++ b/src/main/java/pro/belbix/ethparser/web3/abi/FunctionsNames.java @@ -105,4 +105,5 @@ private FunctionsNames() { public static final String BALANCES = "balances"; public static final String CHECKING_PRICING_TOKEN = "checkPricingToken"; public static final String GET_BALANCES = "get_balances"; + public static final String SLOT_0 = "slot0"; } diff --git a/src/main/java/pro/belbix/ethparser/web3/abi/FunctionsUtils.java b/src/main/java/pro/belbix/ethparser/web3/abi/FunctionsUtils.java index 7801eee9..7c585861 100644 --- a/src/main/java/pro/belbix/ethparser/web3/abi/FunctionsUtils.java +++ b/src/main/java/pro/belbix/ethparser/web3/abi/FunctionsUtils.java @@ -13,6 +13,7 @@ import static pro.belbix.ethparser.web3.abi.FunctionsNames.GET_POOL_TOKENS; import static pro.belbix.ethparser.web3.abi.FunctionsNames.GET_RESERVES; import static pro.belbix.ethparser.web3.abi.FunctionsNames.NAME; +import static pro.belbix.ethparser.web3.abi.FunctionsNames.SLOT_0; import static pro.belbix.ethparser.web3.abi.FunctionsNames.TOKEN0; import static pro.belbix.ethparser.web3.abi.FunctionsNames.TOKEN1; import static pro.belbix.ethparser.web3.contracts.ContractConstants.ZERO_ADDRESS; @@ -140,7 +141,22 @@ public Tuple2 callUniReservesForSwapPlatform(String address, Lon @Cacheable("can_get_token_price") public boolean canGetTokenPrice(String tokenAddress, String oracleAddress, Long block, String network) { return callBoolByNameWithAddressArg(CHECKING_PRICING_TOKEN, tokenAddress, oracleAddress, block, network) - .orElseThrow(() -> new IllegalStateException("Can not call checkPricingToken for address: " + tokenAddress)); + .orElse(false); + } + + @Cacheable("uniswap_slot0") + public Optional getSqrtPriceX96(String address, Long block, String network) { + List types = web3Functions.callFunction(new Function( + SLOT_0, + Collections.emptyList(), + List.of( + new TypeReference() {} + )), address, resolveBlock(block), network); + + if (types == null || types.isEmpty()) { + return Optional.empty(); + } + return Optional.ofNullable(((Uint112)types.get(0)).getValue()); } private Tuple2 callOneInchReserves(String lpAddress, Long block, String network) { diff --git a/src/main/java/pro/belbix/ethparser/web3/contracts/UniswapV3Pools.java b/src/main/java/pro/belbix/ethparser/web3/contracts/UniswapV3Pools.java new file mode 100644 index 00000000..c1829bcf --- /dev/null +++ b/src/main/java/pro/belbix/ethparser/web3/contracts/UniswapV3Pools.java @@ -0,0 +1,129 @@ +package pro.belbix.ethparser.web3.contracts; + +import static pro.belbix.ethparser.service.AbiProviderService.BSC_NETWORK; +import static pro.belbix.ethparser.service.AbiProviderService.ETH_NETWORK; +import static pro.belbix.ethparser.service.AbiProviderService.MATIC_NETWORK; + +import java.util.AbstractMap; +import java.util.List; +import java.util.Map; + +public interface UniswapV3Pools { + + Map> EXCLUDED_STABLE_VAULTS = Map.of( + ETH_NETWORK, List.of( + "0x1851A8fA2ca4d8Fb8B5c56EAC1813Fd890998EFc".toLowerCase(), + "0x7095b06C02B66e4133F7B4b078B2720CB4437408".toLowerCase() + ), + MATIC_NETWORK, List.of(), + BSC_NETWORK, List.of() + ); + + + Map> VAULTS_TO_POOLS = Map.of( + ETH_NETWORK, Map.ofEntries( + new AbstractMap.SimpleEntry<>( + "0xC1aa3966008ef13B9dD2867D41cA21d9C42932A1".toLowerCase(), + "0xb6c52095F2df5967B2028724BC6389f83637f582"), + new AbstractMap.SimpleEntry<>( + "0xf301aF77793322Bbd6C9e7fa4465499cd2bDecDE".toLowerCase(), + "0x8ad599c3A0ff1De082011EFDDc58f1908eb6e6D8"), + new AbstractMap.SimpleEntry<>( + "0x0a1AB972612489a1a362f42559bcd281FBEc0786".toLowerCase(), + "0x4e68Ccd3E89f51C3074ca5072bbAC773960dFa36"), + new AbstractMap.SimpleEntry<>( + "0x5c49E0386215077d1A3eCc425CC30ce34Ec08B60".toLowerCase(), + "0x4e68Ccd3E89f51C3074ca5072bbAC773960dFa36"), + new AbstractMap.SimpleEntry<>( + "0xEA46CfcB43D5274991344cF6F56765e39A7Eae1a".toLowerCase(), + "0x4e68Ccd3E89f51C3074ca5072bbAC773960dFa36"), + new AbstractMap.SimpleEntry<>( + "0xc53DaB6fDD18AF6CD5cF37fDE7C941d368f8664f".toLowerCase(), + "0x4e68Ccd3E89f51C3074ca5072bbAC773960dFa36"), + new AbstractMap.SimpleEntry<>( + "0x65383Abd40f9f831018dF243287F7AE3612c62AC".toLowerCase(), + "0x7379e81228514a1D2a6Cf7559203998E20598346"), + new AbstractMap.SimpleEntry<>( + "0xadB16dF01b9474347E8fffD6032360D3B54627fB".toLowerCase(), + "0xE2de090153403b0F0401142d5394da897630dCb7"), + new AbstractMap.SimpleEntry<>( + "0x2357685B07469eE80A389819C7A41edCD70cd88C".toLowerCase(), + "0xCBCdF9626bC03E24f779434178A73a0B4bad62eD"), + new AbstractMap.SimpleEntry<>( + "0x7Fb7E4aE3a174E24e6Fd545BbF6E9fC5a14162Cc".toLowerCase(), + "0x7Cf82E7b1Ee2a8A5D629F21a720740eCE2f8b7CD"), + new AbstractMap.SimpleEntry<>( + "0xc3426599Ec933FbF657ee44b53e7f01d83Be1f63".toLowerCase(), + "0xfab26CFa923360fFC8ffc40827faeE5500988E9C"), + new AbstractMap.SimpleEntry<>( + "0x45A78dEbfb4d9E94836dC1680d7FAf32b3994a83".toLowerCase(), + "0x8ad599c3A0ff1De082011EFDDc58f1908eb6e6D8"), + new AbstractMap.SimpleEntry<>( + "0x0b4C4EA418Cd596B1204C0dd07E419707149C7C6".toLowerCase(), + "0x8ad599c3A0ff1De082011EFDDc58f1908eb6e6D8"), + new AbstractMap.SimpleEntry<>( + "0xC74075F5c9aD58C655a6160bA955B4aCD5dE8d0B".toLowerCase(), + "0x8ad599c3A0ff1De082011EFDDc58f1908eb6e6D8"), + new AbstractMap.SimpleEntry<>( + "0x3b2ED6013f961404AbA5a030e20A2AceB486832d".toLowerCase(), + "0x8ad599c3A0ff1De082011EFDDc58f1908eb6e6D8"), + new AbstractMap.SimpleEntry<>( + "0xe29385F6B90F25082972B75ccBC69900cE8A176A".toLowerCase(), + "0xEe4Cf3b78A74aFfa38C6a926282bCd8B5952818d"), + new AbstractMap.SimpleEntry<>( + "0x3F16b084Ff94c8a3f5A1b60834046f1febD15595".toLowerCase(), + "0x1d42064Fc4Beb5F8aAF85F4617AE8b3b5B8Bd801"), + new AbstractMap.SimpleEntry<>( + "0xd82964732cF95F904FD814511Be08b86d213232E".toLowerCase(), + "0x3Fae0F474145A1A771F36bD188D1Cc7057A91B06"), + new AbstractMap.SimpleEntry<>( + "0xeC665d477812C11Bf163841C83322FB4743D1Cfa".toLowerCase(), + "0x3Fae0F474145A1A771F36bD188D1Cc7057A91B06"), + + new AbstractMap.SimpleEntry<>( + "0x8e1de189195a76baF8871f70AdcD090aD06a0B58".toLowerCase(), + "0xeD1d49B6BadA7Bb00cC7C7351138BD959cf6B8ba"), + new AbstractMap.SimpleEntry<>( + "0x6bA287890264cFdAb98100E9855b1423328269D2".toLowerCase(), + "0xD43b29AaF8aD938CfF4F478A0756defFfb329D07"), + new AbstractMap.SimpleEntry<>( + "0x50dCcf8F83CCE8aA9168637c2Ec0114ae934F6d1".toLowerCase(), + "0x6BFe36d9a664289cE04F32E4F83f1566c4712F96"), + new AbstractMap.SimpleEntry<>( + "0x04EdB1420A01547944eA57bBd4EBeBAE04ac116b".toLowerCase(), + "0x9359c87B38DD25192c5f2b07b351ac91C90E6ca7"), + new AbstractMap.SimpleEntry<>( + "0x25642078C595A7078f150e5E0486364077aE9eBB".toLowerCase(), + "0xE73FE82f905E265d26e4a5A3D36d0D03bC4119Fc"), + new AbstractMap.SimpleEntry<>( + "0x503Ea79B73995Cf0C8d323C17782047ED5cC72B2".toLowerCase(), + "0xC2e9F25Be6257c210d7Adf0D4Cd6E3E881ba25f8"), + new AbstractMap.SimpleEntry<>( + "0xC905ccc1a1EC21C8bbE0c0b53d3D048D9055D4bB".toLowerCase(), + "0xC2e9F25Be6257c210d7Adf0D4Cd6E3E881ba25f8"), + new AbstractMap.SimpleEntry<>( + "0x970CC1E0Bdb3B29a6A12BDE1954A8509acbC9158".toLowerCase(), + "0xC2e9F25Be6257c210d7Adf0D4Cd6E3E881ba25f8"), + new AbstractMap.SimpleEntry<>( + "0x8137ac6dF358fe2D0DFbB1b5aA87C110950A16Cd".toLowerCase(), + "0xC2e9F25Be6257c210d7Adf0D4Cd6E3E881ba25f8"), + new AbstractMap.SimpleEntry<>( + "0xFb387177fF9Db15294F7Aebb1ea1e941f55695bc".toLowerCase(), + "0xa63b490aA077f541c9d64bFc1Cc0db2a752157b5"), + + // exclude UST_USDT + new AbstractMap.SimpleEntry<>( + "0x1851A8fA2ca4d8Fb8B5c56EAC1813Fd890998EFc".toLowerCase(), + ""), + + // exclued BUSD_USDC + new AbstractMap.SimpleEntry<>( + "0x7095b06C02B66e4133F7B4b078B2720CB4437408".toLowerCase(), + "") + ), + BSC_NETWORK, Map.of(), + MATIC_NETWORK, Map.of() + ); + + +} diff --git a/src/main/java/pro/belbix/ethparser/web3/prices/PriceProvider.java b/src/main/java/pro/belbix/ethparser/web3/prices/PriceProvider.java index 4468d213..9a8cf560 100644 --- a/src/main/java/pro/belbix/ethparser/web3/prices/PriceProvider.java +++ b/src/main/java/pro/belbix/ethparser/web3/prices/PriceProvider.java @@ -8,6 +8,7 @@ import static pro.belbix.ethparser.web3.contracts.ContractConstants.ZERO_ADDRESS; import static pro.belbix.ethparser.web3.contracts.ContractUtils.getBaseNetworkWrappedTokenAddress; +import java.math.BigDecimal; import java.math.BigInteger; import java.util.HashMap; import java.util.HashSet; @@ -32,6 +33,7 @@ import pro.belbix.ethparser.web3.contracts.ContractType; import pro.belbix.ethparser.web3.contracts.ContractUtils; import pro.belbix.ethparser.web3.contracts.UniPairType; +import pro.belbix.ethparser.web3.contracts.UniswapV3Pools; import pro.belbix.ethparser.web3.contracts.db.ContractDbService; @Service @@ -39,6 +41,7 @@ @AllArgsConstructor public class PriceProvider { + private final static BigDecimal UNISWAP_V3_VALUE = BigDecimal.valueOf(2).pow(96).pow(2); private final static Double DEFAULT_RETURN_PRICE = 1D; private final static int DEFAULT_CURVE_SIZE = 3; private final static int DEFAULT_DECIMAL = 18; @@ -185,6 +188,35 @@ public double getLpTokenUsdPriceFromEth(String lpAddress, double amount, long bl return usdValue; } + public double getUniswapV3Price(String address, Long block, String network) { + + if (UniswapV3Pools.EXCLUDED_STABLE_VAULTS.get(network).contains(address.toLowerCase())) { + return 1; + } + + var poolAddress = UniswapV3Pools.VAULTS_TO_POOLS.get(network).get(address.toLowerCase()); + if (poolAddress == null) { + log.error("Can not find uniSwapV3 pool address"); + throw new IllegalStateException(); + } + + var sqrt = functionsUtils.getSqrtPriceX96(poolAddress, block, network) + .orElseThrow(() -> new IllegalStateException("Can not get SqrtPriceX96 for : " + poolAddress)); + + var tokens = functionsUtils.callTokensForSwapPlatform(poolAddress, network); + + var decimalOne = functionsUtils.getDecimal(tokens.component1(), network); + var decimalTwo = functionsUtils.getDecimal(tokens.component2(), network); + + sqrt = sqrt.pow(2); + + var decimal = BigDecimal.valueOf(10).pow(decimalOne).divide(BigDecimal.valueOf(10).pow(decimalTwo)); + var price = sqrt.doubleValue() * decimal.doubleValue() / UNISWAP_V3_VALUE.doubleValue(); + + var tokenTwoPrice = getPriceForCoin(tokens.component2(), block, network); + return price * tokenTwoPrice; + } + private double calculateLpTokenPrice(String lpAddress, Tuple2 lpPooled, double lpBalance, diff --git a/src/test/java/pro/belbix/ethparser/web3/PriceProviderTest.java b/src/test/java/pro/belbix/ethparser/web3/PriceProviderTest.java index 1aa720f3..d65b0404 100644 --- a/src/test/java/pro/belbix/ethparser/web3/PriceProviderTest.java +++ b/src/test/java/pro/belbix/ethparser/web3/PriceProviderTest.java @@ -59,4 +59,16 @@ public void priceForCurve_Curvefi_USD_BTC_ETH() { double price = priceProvider.getCurvePrice("0xc4AD29ba4B3c580e6D59105FFf484999997675Ff", 14354885L, ETH_NETWORK); assertEquals(numberFormat("1459.31"), String.format("%.2f", price)); } + + @Test + public void getUniswapV3Price_zUSD_WETH() { + double price = priceProvider.getUniswapV3Price("0xC1aa3966008ef13B9dD2867D41cA21d9C42932A1", 14522491L, ETH_NETWORK); + assertEquals(numberFormat("0.35"), String.format("%.2f", price)); + } + + @Test + public void getUniswapV3Price_UST_USDT() { + double price = priceProvider.getUniswapV3Price("0x1851A8fA2ca4d8Fb8B5c56EAC1813Fd890998EFc", 14522491L, ETH_NETWORK); + assertEquals(numberFormat("1.00"), String.format("%.2f", price)); + } } From c6d64dafa4c8f4e61e189592a138c188b9e49f0a Mon Sep 17 00:00:00 2001 From: alex Date: Thu, 7 Apr 2022 02:02:12 +0300 Subject: [PATCH 36/65] Parse uniswapv3 event log --- .../task/CovalenthqTransactionTask.java | 37 ++++++--- .../belbix/ethparser/web3/MethodDecoder.java | 12 --- .../belbix/ethparser/web3/SimpleDecoder.java | 80 ++++++++++++++++++- .../contracts/DecodeExcludeConstants.java | 59 ++++++++++++++ .../web3/contracts/UniswapV3Pools.java | 2 - .../ethparser/web3/prices/PriceProvider.java | 4 + .../ethparser/web3/SimpleDecoderTest.java | 45 +++++++++++ 7 files changed, 212 insertions(+), 27 deletions(-) create mode 100644 src/main/java/pro/belbix/ethparser/web3/contracts/DecodeExcludeConstants.java create mode 100644 src/test/java/pro/belbix/ethparser/web3/SimpleDecoderTest.java diff --git a/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java b/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java index 08d53bd8..52558bb7 100644 --- a/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java +++ b/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java @@ -4,7 +4,7 @@ import static pro.belbix.ethparser.entity.profit.CovalenthqVaultTransactionType.DEPOSIT_UNI; import static pro.belbix.ethparser.entity.profit.CovalenthqVaultTransactionType.WITHDRAW; import static pro.belbix.ethparser.entity.profit.CovalenthqVaultTransactionType.WITHDRAW_UNI; -import static pro.belbix.ethparser.service.AbiProviderService.MATIC_NETWORK; +import static pro.belbix.ethparser.service.AbiProviderService.ETH_NETWORK; import java.math.BigDecimal; import java.math.BigInteger; @@ -26,6 +26,7 @@ import org.springframework.data.domain.Sort; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; +import org.web3j.abi.datatypes.Type; import org.web3j.protocol.core.methods.response.Log; import pro.belbix.ethparser.entity.contracts.VaultEntity; import pro.belbix.ethparser.entity.profit.CovalenthqVaultTransaction; @@ -83,13 +84,8 @@ public void start() { log.info("Begin parse vault tx"); var executor = Executors.newFixedThreadPool(maxThreadSize); - vaultRepository.findAllInContractAddress(List.of( - "0x4A5eb54018462803B9D9a35D1730e45D6EE7033E", - "0x8cccdEBF657F072D83B2d94068C4377a3BA91e08", - "0xFf127E0dDB5E0a53d29f9eA029F5b41F281F3EFD", - "0x102Df50dB22407B64a8A6b11734c8743B6AeF953" - ), - MATIC_NETWORK).stream() + vaultRepository.findAllByNetwork(ETH_NETWORK) + .stream() .map(i -> CompletableFuture.runAsync(() -> getVaultTransaction(i), executor)) .collect(Collectors.toList()); @@ -195,7 +191,7 @@ private CovalenthqVaultTransaction toCovalenthqVaultTransaction( log.error("Can not find log event"); throw new IllegalStateException(); }); - var value = getParamValue(item, covalenthqType.value); + var value = getParamValue(item, covalenthqType.value, contractAddress, network); var decimal = item.getContractDecimal() == 0 ? functionsUtils.getDecimal(contractAddress, network) : item.getContractDecimal(); @@ -206,7 +202,7 @@ private CovalenthqVaultTransaction toCovalenthqVaultTransaction( transaction.setContractDecimal(decimal); transaction.setContractAddress(contractAddress); transaction.setType(covalenthqType.type); - transaction.setOwnerAddress(getParamValue(item, covalenthqType.address)); + transaction.setOwnerAddress(getParamValue(item, covalenthqType.address, contractAddress, network)); transaction.setValue( StringUtils.isEmpty(value) ? BigDecimal.ZERO : new BigDecimal(value) ); @@ -228,14 +224,31 @@ private CovalenthqVaultTransactionType toCovalenthqVaultTransactionType(Covalent return null; } - private String getParamValue(CovalenthqTransactionHistoryItemLog item, String param) { + private String getParamValue(CovalenthqTransactionHistoryItemLog item, String param, String address, String network) { var ethLog = new Log(); ethLog.setTopics(item.getTopics()); ethLog.setData(item.getData()); - var result = simpleDecoder.decodeEthLog(ethLog) + List result = simpleDecoder.decodeEthLogForDepositAndWithdraw(ethLog, address, network) .orElseThrow(); + +// if (DecodeExcludeConstants.DECODE_UNISWAP_V3_EVENT.get(network).contains(address.toLowerCase())) { +// result = simpleDecoder.decodeUniswapV3EthLog(ethLog); +// } else { +// result = simpleDecoder.decodeEthLog(ethLog) +// .orElseThrow(); +// +// if (result.isEmpty() +// && ethLog.getData() == null +// && DecodeExcludeConstants.DECODE_ONLY_TOPICS.containsKey(address.toLowerCase()) +// ) { +// log.error("Try to decode only topics - {}", ethLog); +// ethLog.setData(StringUtils.EMPTY); +// result = simpleDecoder.decodeOnlyTopics(ethLog); +// } +// } + var indexParam = -1; var castToString = false; diff --git a/src/main/java/pro/belbix/ethparser/web3/MethodDecoder.java b/src/main/java/pro/belbix/ethparser/web3/MethodDecoder.java index 19031ae1..d026043f 100644 --- a/src/main/java/pro/belbix/ethparser/web3/MethodDecoder.java +++ b/src/main/java/pro/belbix/ethparser/web3/MethodDecoder.java @@ -97,18 +97,6 @@ public static List extractLogIndexedValues( indexedValues.addAll(nonIndexedValues); return indexedValues; } - public static List extractLogIndexedValuesWrapped( - List topics, - String data, - List> parameters) { - try { - return extractLogIndexedValues(topics, data, parameters); - } catch (Exception e) { - log.error("Get error during parse log: {}", e.getMessage()); - // in some cases parameters in data - return FunctionReturnDecoder.decode(data, parameters); - } - } public static List> getNonIndexedParameters( List> parameters) { diff --git a/src/main/java/pro/belbix/ethparser/web3/SimpleDecoder.java b/src/main/java/pro/belbix/ethparser/web3/SimpleDecoder.java index b74a97bc..fbb1ecae 100644 --- a/src/main/java/pro/belbix/ethparser/web3/SimpleDecoder.java +++ b/src/main/java/pro/belbix/ethparser/web3/SimpleDecoder.java @@ -1,24 +1,68 @@ package pro.belbix.ethparser.web3; +import static org.web3j.abi.FunctionReturnDecoder.decodeIndexedValue; + +import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.TreeMap; import java.util.function.Predicate; +import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; +import org.web3j.abi.FunctionReturnDecoder; +import org.web3j.abi.TypeReference; import org.web3j.abi.datatypes.Type; import org.web3j.protocol.core.methods.response.EthLog.LogResult; import org.web3j.protocol.core.methods.response.Log; import org.web3j.protocol.core.methods.response.Transaction; import pro.belbix.ethparser.model.tx.EthTransactionI; +import pro.belbix.ethparser.web3.contracts.DecodeExcludeConstants; @Component +@Slf4j public class SimpleDecoder extends MethodDecoder { public Optional> decodeEthLog(Log ethLog) { return parseMethodId(ethLog) .flatMap(this::findParameters) .map(parameters -> - extractLogIndexedValuesWrapped(ethLog.getTopics(), ethLog.getData(), parameters)); + extractLogIndexedValues(ethLog.getTopics(), ethLog.getData(), parameters)); + } + + public Optional> decodeEthLogForDepositAndWithdraw(Log ethLog, String address, String network) { + return parseMethodId(ethLog) + .flatMap(this::findParameters) + .map(parameters -> + extractLogIndexedValuesWrapped(ethLog.getTopics(), ethLog.getData(), parameters, address, network)); + } + + // deposit and withdraw + public List decodeOnlyTopics(List topics, String data) { + try { + return extractLogIndexedValues(topics, data, List.of( + TypeReference.makeTypeReference("address", true, false), + TypeReference.makeTypeReference("uint256", true, false) + )); + } catch (Exception e) { + log.error("Error during decodeOnlyTopics", e); + return List.of(); + } + } + + public List extractLogIndexedValuesWrapped( + List topics, + String data, + List> parameters, String address, String network) { + try { + return extractLogIndexedValues(topics, data, parameters); + } catch (Exception e) { + log.error("Get error during parse log: {}", e.getMessage()); + if (data == null + && DecodeExcludeConstants.DECODE_ONLY_TOPICS.containsKey(address.toLowerCase())) { + return decodeOnlyTopics(topics, data); + } + return extractLogIndexValues(topics, data); + } } @SuppressWarnings("rawtypes") @@ -43,4 +87,38 @@ public EthTransactionI mapTypesToModel(List types, String methodID, Transaction transaction) { throw new UnsupportedOperationException(); } + + // method only for deposit and withdraw + private List extractLogIndexValues(List topics, String data) { + try { + var values = new ArrayList(); + + List> parameters = List.of( + TypeReference.makeTypeReference("uint256"), + TypeReference.makeTypeReference("uint256") + ); + + if (data == null) { + return values; + } + + if (topics == null || topics.size() < 2) { + log.error("Can not extract logs for uniswapv3, topics - {}", topics); + return values; + } + + values.add( + decodeIndexedValue(topics.get(1), TypeReference.makeTypeReference("address", true, false)) + ); + + values.addAll( + FunctionReturnDecoder.decode(data, getNonIndexedParameters(parameters)) + ); + + return values; + } catch (Exception e) { + log.error("Can not extract logs for uniswapv3", e); + throw new IllegalStateException(e); + } + } } diff --git a/src/main/java/pro/belbix/ethparser/web3/contracts/DecodeExcludeConstants.java b/src/main/java/pro/belbix/ethparser/web3/contracts/DecodeExcludeConstants.java new file mode 100644 index 00000000..cc1a0b0e --- /dev/null +++ b/src/main/java/pro/belbix/ethparser/web3/contracts/DecodeExcludeConstants.java @@ -0,0 +1,59 @@ +package pro.belbix.ethparser.web3.contracts; + +import static pro.belbix.ethparser.service.AbiProviderService.BSC_NETWORK; +import static pro.belbix.ethparser.service.AbiProviderService.ETH_NETWORK; +import static pro.belbix.ethparser.service.AbiProviderService.MATIC_NETWORK; + +import java.util.List; +import java.util.Map; + +public interface DecodeExcludeConstants { + Map> DECODE_UNISWAP_V3_EVENT = Map.of( + ETH_NETWORK, List.of( + "0xC1aa3966008ef13B9dD2867D41cA21d9C42932A1".toLowerCase(), + "0xf301aF77793322Bbd6C9e7fa4465499cd2bDecDE".toLowerCase(), + "0x0a1AB972612489a1a362f42559bcd281FBEc0786".toLowerCase(), + "0x5c49E0386215077d1A3eCc425CC30ce34Ec08B60".toLowerCase(), + "0xEA46CfcB43D5274991344cF6F56765e39A7Eae1a".toLowerCase(), + "0xc53DaB6fDD18AF6CD5cF37fDE7C941d368f8664f".toLowerCase(), + "0x65383Abd40f9f831018dF243287F7AE3612c62AC".toLowerCase(), + "0xadB16dF01b9474347E8fffD6032360D3B54627fB".toLowerCase(), + "0x2357685B07469eE80A389819C7A41edCD70cd88C".toLowerCase(), + "0x7Fb7E4aE3a174E24e6Fd545BbF6E9fC5a14162Cc".toLowerCase(), + "0xc3426599Ec933FbF657ee44b53e7f01d83Be1f63".toLowerCase(), + "0x45A78dEbfb4d9E94836dC1680d7FAf32b3994a83".toLowerCase(), + "0x0b4C4EA418Cd596B1204C0dd07E419707149C7C6".toLowerCase(), + "0xC74075F5c9aD58C655a6160bA955B4aCD5dE8d0B".toLowerCase(), + "0x3b2ED6013f961404AbA5a030e20A2AceB486832d".toLowerCase(), + "0xe29385F6B90F25082972B75ccBC69900cE8A176A".toLowerCase(), + "0x3F16b084Ff94c8a3f5A1b60834046f1febD15595".toLowerCase(), + "0xd82964732cF95F904FD814511Be08b86d213232E".toLowerCase(), + "0xeC665d477812C11Bf163841C83322FB4743D1Cfa".toLowerCase(), + "0x8e1de189195a76baF8871f70AdcD090aD06a0B58".toLowerCase(), + "0x6bA287890264cFdAb98100E9855b1423328269D2".toLowerCase(), + "0x50dCcf8F83CCE8aA9168637c2Ec0114ae934F6d1".toLowerCase(), + "0x04EdB1420A01547944eA57bBd4EBeBAE04ac116b".toLowerCase(), + "0x25642078C595A7078f150e5E0486364077aE9eBB".toLowerCase(), + "0x503Ea79B73995Cf0C8d323C17782047ED5cC72B2".toLowerCase(), + "0xC905ccc1a1EC21C8bbE0c0b53d3D048D9055D4bB".toLowerCase(), + "0x970CC1E0Bdb3B29a6A12BDE1954A8509acbC9158".toLowerCase(), + "0x8137ac6dF358fe2D0DFbB1b5aA87C110950A16Cd".toLowerCase(), + "0xFb387177fF9Db15294F7Aebb1ea1e941f55695bc".toLowerCase(), + "0x1851A8fA2ca4d8Fb8B5c56EAC1813Fd890998EFc".toLowerCase(), + "0x7095b06C02B66e4133F7B4b078B2720CB4437408".toLowerCase(), + "0x744F77705749541926294f7b35A63f8691374640".toLowerCase(), + "0x41d0Ab2Bf4E8Da44FEE38232E56b089f3Bb00587".toLowerCase() + ), + BSC_NETWORK, List.of(), + MATIC_NETWORK, List.of() + ); + + Map> DECODE_ONLY_TOPICS = Map.of( + ETH_NETWORK, List.of( + // iFARM + "0x1571eD0bed4D987fe2b498DdBaE7DFA19519F651".toLowerCase() + ), + BSC_NETWORK, List.of(), + MATIC_NETWORK, List.of() + ); +} diff --git a/src/main/java/pro/belbix/ethparser/web3/contracts/UniswapV3Pools.java b/src/main/java/pro/belbix/ethparser/web3/contracts/UniswapV3Pools.java index c1829bcf..38bfa3e2 100644 --- a/src/main/java/pro/belbix/ethparser/web3/contracts/UniswapV3Pools.java +++ b/src/main/java/pro/belbix/ethparser/web3/contracts/UniswapV3Pools.java @@ -124,6 +124,4 @@ public interface UniswapV3Pools { BSC_NETWORK, Map.of(), MATIC_NETWORK, Map.of() ); - - } diff --git a/src/main/java/pro/belbix/ethparser/web3/prices/PriceProvider.java b/src/main/java/pro/belbix/ethparser/web3/prices/PriceProvider.java index 9a8cf560..afdabef7 100644 --- a/src/main/java/pro/belbix/ethparser/web3/prices/PriceProvider.java +++ b/src/main/java/pro/belbix/ethparser/web3/prices/PriceProvider.java @@ -142,6 +142,10 @@ public double getCurvePrice(String address, Long block, String network) { throw new CanNotFetchPriceException(); }); + if (tokenInfo.getAddress().equalsIgnoreCase(ZERO_ADDRESS)) { + return 1; + } + var tokenDecimal = functionsUtils.callIntByName(FunctionsNames.DECIMALS, tokenInfo.getAddress(), block, network) .orElseThrow(() -> { log.error("Can not get decimal for {} {}", tokenInfo.getAddress(), network); diff --git a/src/test/java/pro/belbix/ethparser/web3/SimpleDecoderTest.java b/src/test/java/pro/belbix/ethparser/web3/SimpleDecoderTest.java new file mode 100644 index 00000000..b3c295dc --- /dev/null +++ b/src/test/java/pro/belbix/ethparser/web3/SimpleDecoderTest.java @@ -0,0 +1,45 @@ +package pro.belbix.ethparser.web3; + +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; +import static pro.belbix.ethparser.service.AbiProviderService.ETH_NETWORK; + +import java.math.BigInteger; +import java.util.List; +import org.junit.jupiter.api.Test; +import org.web3j.protocol.core.methods.response.Log; + +public class SimpleDecoderTest { + + SimpleDecoder decoder = new SimpleDecoder(); + + + @Test + public void decodeUniswapV3EthLogTest_deposit() { + var log = new Log(); + log.setTopics(List.of( + "0x90890809c654f11d6e72a28fa60149770a0d11ec6c92319d6ceb2bb0a4ea1a15", + "0x000000000000000000000000b45844826d2757e5bc43518c084e9774d6715f4a" + )); + log.setData("0x00000000000000000000000000000000000000000000005053b0b935131f65ee00000000000000000000000000000000000000000000000000000000578f2940"); + + var result = decoder.decodeEthLogForDepositAndWithdraw(log, "", ETH_NETWORK).orElseThrow(); + + assertThat(result).isNotNull(); + assertThat(result.size()).isEqualTo(3); + assertThat(result.get(2).getValue()).isEqualTo(BigInteger.valueOf(1469000000)); + } + + // txHash = 0x318bd2dff3c071b883cbaee68bde0a306f704c445ceea404916546fbc456199b + @Test + public void decodeOnlyTopics_deposit() { + + var result = decoder.decodeOnlyTopics(List.of( + "0xe1fffcc4923d04b559f4d29a8bfc6cda04eb5b0d3c460751c2402c5c5cc9109c", + "0x0000000000000000000000008369e7900ff2359bb36ef1c40a60e5f76373a6ed", + "0x0000000000000000000000000000000000000000000000018493fba64ef00000" + ), ""); + assertThat(result).isNotNull(); + assertThat(result.size()).isEqualTo(2); + assertThat(result.get(1).getValue()).isEqualTo(new BigInteger("28000000000000000000")); + } +} From 44078ee0c220e5d7f48a21aac6e5ad593fe6fd9a Mon Sep 17 00:00:00 2001 From: alex Date: Fri, 8 Apr 2022 11:56:24 +0300 Subject: [PATCH 37/65] Fix problem with curve --- .../task/CovalenthqTransactionTask.java | 22 +++++-------------- .../belbix/ethparser/web3/SimpleDecoder.java | 15 +++++-------- .../web3/contracts/ContractConstants.java | 1 + .../ethparser/web3/prices/PriceProvider.java | 10 ++++----- .../ethparser/web3/SimpleDecoderTest.java | 12 +++++----- 5 files changed, 24 insertions(+), 36 deletions(-) diff --git a/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java b/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java index 52558bb7..6fb9b8d2 100644 --- a/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java +++ b/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java @@ -42,6 +42,7 @@ import pro.belbix.ethparser.web3.SimpleDecoder; import pro.belbix.ethparser.web3.Web3Functions; import pro.belbix.ethparser.web3.abi.FunctionsUtils; +import pro.belbix.ethparser.web3.contracts.DecodeExcludeConstants; @Service @RequiredArgsConstructor @@ -229,25 +230,14 @@ private String getParamValue(CovalenthqTransactionHistoryItemLog item, String pa ethLog.setTopics(item.getTopics()); ethLog.setData(item.getData()); - List result = simpleDecoder.decodeEthLogForDepositAndWithdraw(ethLog, address, network) + List result = simpleDecoder.decodeEthLogForDepositAndWithdraw(ethLog) .orElseThrow(); + if (ethLog.getData() == null && DecodeExcludeConstants.DECODE_ONLY_TOPICS.get(ETH_NETWORK).contains(address.toLowerCase())) { + ethLog.setData(""); + result = simpleDecoder.decodeOnlyTopics(ethLog); + } -// if (DecodeExcludeConstants.DECODE_UNISWAP_V3_EVENT.get(network).contains(address.toLowerCase())) { -// result = simpleDecoder.decodeUniswapV3EthLog(ethLog); -// } else { -// result = simpleDecoder.decodeEthLog(ethLog) -// .orElseThrow(); -// -// if (result.isEmpty() -// && ethLog.getData() == null -// && DecodeExcludeConstants.DECODE_ONLY_TOPICS.containsKey(address.toLowerCase()) -// ) { -// log.error("Try to decode only topics - {}", ethLog); -// ethLog.setData(StringUtils.EMPTY); -// result = simpleDecoder.decodeOnlyTopics(ethLog); -// } -// } var indexParam = -1; var castToString = false; diff --git a/src/main/java/pro/belbix/ethparser/web3/SimpleDecoder.java b/src/main/java/pro/belbix/ethparser/web3/SimpleDecoder.java index fbb1ecae..e1559fc2 100644 --- a/src/main/java/pro/belbix/ethparser/web3/SimpleDecoder.java +++ b/src/main/java/pro/belbix/ethparser/web3/SimpleDecoder.java @@ -16,7 +16,6 @@ import org.web3j.protocol.core.methods.response.Log; import org.web3j.protocol.core.methods.response.Transaction; import pro.belbix.ethparser.model.tx.EthTransactionI; -import pro.belbix.ethparser.web3.contracts.DecodeExcludeConstants; @Component @Slf4j @@ -29,17 +28,17 @@ public Optional> decodeEthLog(Log ethLog) { extractLogIndexedValues(ethLog.getTopics(), ethLog.getData(), parameters)); } - public Optional> decodeEthLogForDepositAndWithdraw(Log ethLog, String address, String network) { + public Optional> decodeEthLogForDepositAndWithdraw(Log ethLog) { return parseMethodId(ethLog) .flatMap(this::findParameters) .map(parameters -> - extractLogIndexedValuesWrapped(ethLog.getTopics(), ethLog.getData(), parameters, address, network)); + extractLogIndexedValuesWrapped(ethLog.getTopics(), ethLog.getData(), parameters)); } // deposit and withdraw - public List decodeOnlyTopics(List topics, String data) { + public List decodeOnlyTopics(Log ethLog) { try { - return extractLogIndexedValues(topics, data, List.of( + return extractLogIndexedValues(ethLog.getTopics(), ethLog.getData(), List.of( TypeReference.makeTypeReference("address", true, false), TypeReference.makeTypeReference("uint256", true, false) )); @@ -52,15 +51,11 @@ public List decodeOnlyTopics(List topics, String data) { public List extractLogIndexedValuesWrapped( List topics, String data, - List> parameters, String address, String network) { + List> parameters) { try { return extractLogIndexedValues(topics, data, parameters); } catch (Exception e) { log.error("Get error during parse log: {}", e.getMessage()); - if (data == null - && DecodeExcludeConstants.DECODE_ONLY_TOPICS.containsKey(address.toLowerCase())) { - return decodeOnlyTopics(topics, data); - } return extractLogIndexValues(topics, data); } } diff --git a/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstants.java b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstants.java index 0ad35bed..e2b51582 100644 --- a/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstants.java +++ b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstants.java @@ -48,6 +48,7 @@ public class ContractConstants { static final String ONE_INCH_FACTORY_BSC = "0xD41B24bbA51fAc0E4827b6F94C0D6DDeB183cD64" .toLowerCase(); public static final String ZERO_ADDRESS = "0x0000000000000000000000000000000000000000"; + public static final String CURVE_ZERO_ADDRESS = "0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee"; public static final String FARM_TOKEN = "0xa0246c9032bc3a600820415ae600c6388619a14d" .toLowerCase(); public static final String BSC_FARM_TOKEN = "0x4B5C23cac08a567ecf0c1fFcA8372A45a5D33743" diff --git a/src/main/java/pro/belbix/ethparser/web3/prices/PriceProvider.java b/src/main/java/pro/belbix/ethparser/web3/prices/PriceProvider.java index afdabef7..ef71ff82 100644 --- a/src/main/java/pro/belbix/ethparser/web3/prices/PriceProvider.java +++ b/src/main/java/pro/belbix/ethparser/web3/prices/PriceProvider.java @@ -5,6 +5,7 @@ import static pro.belbix.ethparser.web3.abi.FunctionsNames.MINTER; import static pro.belbix.ethparser.web3.abi.FunctionsNames.NAME; import static pro.belbix.ethparser.web3.abi.FunctionsNames.TOTAL_SUPPLY; +import static pro.belbix.ethparser.web3.contracts.ContractConstants.CURVE_ZERO_ADDRESS; import static pro.belbix.ethparser.web3.contracts.ContractConstants.ZERO_ADDRESS; import static pro.belbix.ethparser.web3.contracts.ContractUtils.getBaseNetworkWrappedTokenAddress; @@ -146,6 +147,10 @@ public double getCurvePrice(String address, Long block, String network) { return 1; } + if (tokenInfo.getAddress().equalsIgnoreCase(CURVE_ZERO_ADDRESS)) { + tokenInfo.setAddress(ContractUtils.getBaseNetworkWrappedTokenAddress(network)); + } + var tokenDecimal = functionsUtils.callIntByName(FunctionsNames.DECIMALS, tokenInfo.getAddress(), block, network) .orElseThrow(() -> { log.error("Can not get decimal for {} {}", tokenInfo.getAddress(), network); @@ -372,11 +377,6 @@ private double getPriceForCoinFromEthLegacy( if (curveUnderlying != null) { return getPriceForCoinFromEthLegacy(curveUnderlying, block, network, handled); } -// -// var vaultName = functionsUtils.getName(address, network); -// if (UniPairType.isCurve(vaultName)) { -// return getCurvePrice(address, block, network); -// } } catch (Exception ignore) { } log.error("Not found lp for {}, block: {}, network: {}", address, block, network); diff --git a/src/test/java/pro/belbix/ethparser/web3/SimpleDecoderTest.java b/src/test/java/pro/belbix/ethparser/web3/SimpleDecoderTest.java index b3c295dc..81e7c5ef 100644 --- a/src/test/java/pro/belbix/ethparser/web3/SimpleDecoderTest.java +++ b/src/test/java/pro/belbix/ethparser/web3/SimpleDecoderTest.java @@ -1,7 +1,6 @@ package pro.belbix.ethparser.web3; import static org.assertj.core.api.AssertionsForClassTypes.assertThat; -import static pro.belbix.ethparser.service.AbiProviderService.ETH_NETWORK; import java.math.BigInteger; import java.util.List; @@ -22,7 +21,7 @@ public void decodeUniswapV3EthLogTest_deposit() { )); log.setData("0x00000000000000000000000000000000000000000000005053b0b935131f65ee00000000000000000000000000000000000000000000000000000000578f2940"); - var result = decoder.decodeEthLogForDepositAndWithdraw(log, "", ETH_NETWORK).orElseThrow(); + var result = decoder.decodeEthLogForDepositAndWithdraw(log).orElseThrow(); assertThat(result).isNotNull(); assertThat(result.size()).isEqualTo(3); @@ -32,12 +31,15 @@ public void decodeUniswapV3EthLogTest_deposit() { // txHash = 0x318bd2dff3c071b883cbaee68bde0a306f704c445ceea404916546fbc456199b @Test public void decodeOnlyTopics_deposit() { - - var result = decoder.decodeOnlyTopics(List.of( + var log = new Log(); + log.setTopics(List.of( "0xe1fffcc4923d04b559f4d29a8bfc6cda04eb5b0d3c460751c2402c5c5cc9109c", "0x0000000000000000000000008369e7900ff2359bb36ef1c40a60e5f76373a6ed", "0x0000000000000000000000000000000000000000000000018493fba64ef00000" - ), ""); + )); + log.setData(""); + + var result = decoder.decodeOnlyTopics(log); assertThat(result).isNotNull(); assertThat(result.size()).isEqualTo(2); assertThat(result.get(1).getValue()).isEqualTo(new BigInteger("28000000000000000000")); From 4657e5074d8faa2bc3c65244f4d5729ffe8786e5 Mon Sep 17 00:00:00 2001 From: alex Date: Sun, 10 Apr 2022 01:20:22 +0300 Subject: [PATCH 38/65] Fix problem with iFarm vault and try to update vaults and pools from harvest API --- .../belbix/ethparser/model/CovalenthqTransactionHistory.java | 5 +++++ src/main/resources/application.yml | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/main/java/pro/belbix/ethparser/model/CovalenthqTransactionHistory.java b/src/main/java/pro/belbix/ethparser/model/CovalenthqTransactionHistory.java index aa539701..e0823204 100644 --- a/src/main/java/pro/belbix/ethparser/model/CovalenthqTransactionHistory.java +++ b/src/main/java/pro/belbix/ethparser/model/CovalenthqTransactionHistory.java @@ -1,6 +1,7 @@ package pro.belbix.ethparser.model; import com.fasterxml.jackson.annotation.JsonFormat; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonSerialize; @@ -11,15 +12,18 @@ import lombok.Data; @Data +@JsonIgnoreProperties public class CovalenthqTransactionHistory { private CovalenthqTransactionHistoryItems data; @Data + @JsonIgnoreProperties public static class CovalenthqTransactionHistoryItems { private List items; private CovalenthqTransactionHistoryPagination pagination; @Data + @JsonIgnoreProperties public static class CovalenthqTransactionHistoryItem { @JsonProperty("block_height") private long blockHeight; @@ -33,6 +37,7 @@ public static class CovalenthqTransactionHistoryItem { private List logs; @Data + @JsonIgnoreProperties public static class CovalenthqTransactionHistoryItemLog { @JsonProperty("block_signed_at") @JsonSerialize(using = LocalDateTimeSerializer.class) diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index be960eaf..c82f4889 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -71,10 +71,10 @@ external: task: pool: fixedRate: 3600000 # Each hour - enable: false + enable: true vault: fixedRate: 3600000 # Each hour - enable: false + enable: true uni-pair: fixedRate: 86400000 # Everyday enable: false From fa5d33fe4c504960c18fadff2463de71f2fcca09 Mon Sep 17 00:00:00 2001 From: alex Date: Sun, 10 Apr 2022 01:34:25 +0300 Subject: [PATCH 39/65] Try fix problem with save --- .../pro/belbix/ethparser/entity/contracts/ContractEntity.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/pro/belbix/ethparser/entity/contracts/ContractEntity.java b/src/main/java/pro/belbix/ethparser/entity/contracts/ContractEntity.java index 1f9fd928..80331d55 100644 --- a/src/main/java/pro/belbix/ethparser/entity/contracts/ContractEntity.java +++ b/src/main/java/pro/belbix/ethparser/entity/contracts/ContractEntity.java @@ -1,6 +1,8 @@ package pro.belbix.ethparser.entity.contracts; import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; import javax.persistence.Id; import javax.persistence.Index; import javax.persistence.Table; @@ -25,7 +27,7 @@ public class ContractEntity { @Id // TODO Can not save for HarvestVaultInfoTask, after return GeneratedValue annotation -// @GeneratedValue(strategy = GenerationType.IDENTITY) + @GeneratedValue(strategy = GenerationType.IDENTITY) private Integer id; private String address; private String name; From 67c8d049945e1b72641d81f6d6e0ac5e9e7a5884 Mon Sep 17 00:00:00 2001 From: alex Date: Sun, 10 Apr 2022 01:54:54 +0300 Subject: [PATCH 40/65] Added logs for check id --- .../pro/belbix/ethparser/service/task/HarvestVaultInfoTask.java | 1 + src/main/resources/application.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/pro/belbix/ethparser/service/task/HarvestVaultInfoTask.java b/src/main/java/pro/belbix/ethparser/service/task/HarvestVaultInfoTask.java index b744c130..37851c95 100644 --- a/src/main/java/pro/belbix/ethparser/service/task/HarvestVaultInfoTask.java +++ b/src/main/java/pro/belbix/ethparser/service/task/HarvestVaultInfoTask.java @@ -104,6 +104,7 @@ private List doTaskByAddressAndNetwork(Map Date: Sun, 10 Apr 2022 01:59:24 +0300 Subject: [PATCH 41/65] Added logs for check id --- .../pro/belbix/ethparser/service/task/HarvestVaultInfoTask.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/pro/belbix/ethparser/service/task/HarvestVaultInfoTask.java b/src/main/java/pro/belbix/ethparser/service/task/HarvestVaultInfoTask.java index 37851c95..dc6453b5 100644 --- a/src/main/java/pro/belbix/ethparser/service/task/HarvestVaultInfoTask.java +++ b/src/main/java/pro/belbix/ethparser/service/task/HarvestVaultInfoTask.java @@ -104,7 +104,7 @@ private List doTaskByAddressAndNetwork(Map Date: Sun, 10 Apr 2022 02:13:33 +0300 Subject: [PATCH 42/65] Try resolve problem with max id for contract --- .../service/task/HarvestVaultInfoTask.java | 34 ++++++++++--------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/src/main/java/pro/belbix/ethparser/service/task/HarvestVaultInfoTask.java b/src/main/java/pro/belbix/ethparser/service/task/HarvestVaultInfoTask.java index dc6453b5..51b6fc77 100644 --- a/src/main/java/pro/belbix/ethparser/service/task/HarvestVaultInfoTask.java +++ b/src/main/java/pro/belbix/ethparser/service/task/HarvestVaultInfoTask.java @@ -103,22 +103,24 @@ private List doTaskByAddressAndNetwork(Map Date: Sun, 10 Apr 2022 02:19:08 +0300 Subject: [PATCH 43/65] Try resolve problem with max id for contract --- .../entity/contracts/ContractEntity.java | 4 +-- .../service/task/HarvestVaultInfoTask.java | 34 +++++++++---------- 2 files changed, 17 insertions(+), 21 deletions(-) diff --git a/src/main/java/pro/belbix/ethparser/entity/contracts/ContractEntity.java b/src/main/java/pro/belbix/ethparser/entity/contracts/ContractEntity.java index 80331d55..1f9fd928 100644 --- a/src/main/java/pro/belbix/ethparser/entity/contracts/ContractEntity.java +++ b/src/main/java/pro/belbix/ethparser/entity/contracts/ContractEntity.java @@ -1,8 +1,6 @@ package pro.belbix.ethparser.entity.contracts; import javax.persistence.Entity; -import javax.persistence.GeneratedValue; -import javax.persistence.GenerationType; import javax.persistence.Id; import javax.persistence.Index; import javax.persistence.Table; @@ -27,7 +25,7 @@ public class ContractEntity { @Id // TODO Can not save for HarvestVaultInfoTask, after return GeneratedValue annotation - @GeneratedValue(strategy = GenerationType.IDENTITY) +// @GeneratedValue(strategy = GenerationType.IDENTITY) private Integer id; private String address; private String name; diff --git a/src/main/java/pro/belbix/ethparser/service/task/HarvestVaultInfoTask.java b/src/main/java/pro/belbix/ethparser/service/task/HarvestVaultInfoTask.java index 51b6fc77..dc6453b5 100644 --- a/src/main/java/pro/belbix/ethparser/service/task/HarvestVaultInfoTask.java +++ b/src/main/java/pro/belbix/ethparser/service/task/HarvestVaultInfoTask.java @@ -103,24 +103,22 @@ private List doTaskByAddressAndNetwork(Map Date: Sun, 10 Apr 2022 02:55:55 +0300 Subject: [PATCH 44/65] Try resolve problem with save contract --- .../pro/belbix/ethparser/web3/contracts/ContractLoader.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/main/java/pro/belbix/ethparser/web3/contracts/ContractLoader.java b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractLoader.java index e546bdd6..5bbe75dd 100644 --- a/src/main/java/pro/belbix/ethparser/web3/contracts/ContractLoader.java +++ b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractLoader.java @@ -1,5 +1,6 @@ package pro.belbix.ethparser.web3.contracts; +import static pro.belbix.ethparser.service.AbiProviderService.ETH_NETWORK; import static pro.belbix.ethparser.web3.abi.FunctionsNames.CONTROLLER; import static pro.belbix.ethparser.web3.abi.FunctionsNames.FACTORY; import static pro.belbix.ethparser.web3.abi.FunctionsNames.GOVERNANCE; @@ -14,8 +15,6 @@ import static pro.belbix.ethparser.web3.contracts.ContractConstants.PAIR_TYPE_ONEINCHE; import static pro.belbix.ethparser.web3.contracts.ContractConstants.PAIR_TYPE_SUSHI; import static pro.belbix.ethparser.web3.contracts.ContractConstants.PAIR_TYPE_UNISWAP; -import static pro.belbix.ethparser.web3.contracts.ContractConstants.PS_ADDRESS; -import static pro.belbix.ethparser.web3.contracts.ContractConstants.PS_V0_ADDRESS; import static pro.belbix.ethparser.web3.contracts.ContractConstants.SUSHISWAP_FACTORY_ADDRESS; import static pro.belbix.ethparser.web3.contracts.ContractConstants.UNISWAP_FACTORY_ADDRESS; import static pro.belbix.ethparser.web3.contracts.ContractConstants.ZERO_ADDRESS; @@ -250,8 +249,7 @@ public void enrichVault(VaultEntity vaultEntity, Long block, Long blockForProper network )); //exclude PS vaults - if (address.equalsIgnoreCase(PS_ADDRESS) - || address.equalsIgnoreCase(PS_V0_ADDRESS)) { + if (ContractConstantsV2.EXCLUDE_ADDRESSES_FOR_PRICE_SHARE_BY_NETWORK.get(ETH_NETWORK).contains(address.toLowerCase())) { vaultEntity.setName("PS vault"); vaultEntity.setDecimals(18L); return; From 148f286fd7185cfd63bc7c50700800cf4d02e13c Mon Sep 17 00:00:00 2001 From: alex Date: Sun, 10 Apr 2022 03:16:41 +0300 Subject: [PATCH 45/65] Try resolve problem with save vault --- .../java/pro/belbix/ethparser/entity/contracts/VaultEntity.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/pro/belbix/ethparser/entity/contracts/VaultEntity.java b/src/main/java/pro/belbix/ethparser/entity/contracts/VaultEntity.java index f3cecb59..3382ece6 100644 --- a/src/main/java/pro/belbix/ethparser/entity/contracts/VaultEntity.java +++ b/src/main/java/pro/belbix/ethparser/entity/contracts/VaultEntity.java @@ -22,7 +22,7 @@ public class VaultEntity { @Id - @GeneratedValue(strategy = GenerationType.IDENTITY) + @GeneratedValue(strategy = GenerationType.SEQUENCE) private Integer id; @OneToOne(fetch = FetchType.EAGER) @JoinColumn(name = "contract", unique = true) From d9e4b45374b64a2906902620ed2bdc34e3a8ce14 Mon Sep 17 00:00:00 2001 From: alex Date: Sun, 10 Apr 2022 13:08:49 +0300 Subject: [PATCH 46/65] Added log --- .../pro/belbix/ethparser/service/task/HarvestVaultInfoTask.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/pro/belbix/ethparser/service/task/HarvestVaultInfoTask.java b/src/main/java/pro/belbix/ethparser/service/task/HarvestVaultInfoTask.java index dc6453b5..1f029d08 100644 --- a/src/main/java/pro/belbix/ethparser/service/task/HarvestVaultInfoTask.java +++ b/src/main/java/pro/belbix/ethparser/service/task/HarvestVaultInfoTask.java @@ -83,7 +83,7 @@ private List doTaskByAddressAndNetwork(Map i.getVaultAddress().toLowerCase()).collect(Collectors.toList()), ContractType.VAULT.getId()); - + log.info("Exist vaults {}", existVaults); var notSavedVaults = items.values().stream() .filter(i -> existVaults.stream().filter(c -> c.getAddress().equalsIgnoreCase(i.getVaultAddress())).findFirst().isEmpty()) .collect(Collectors.toList()); From fe6c8d48414751728c5d906678b6db7ed89d596c Mon Sep 17 00:00:00 2001 From: alex Date: Mon, 11 Apr 2022 02:59:02 +0300 Subject: [PATCH 47/65] Fix problem with decode logs --- .../entity/contracts/VaultEntity.java | 2 +- .../CovalenthqVaultTransactionType.java | 12 ++++--- .../task/CovalenthqTransactionTask.java | 11 +++---- .../service/task/HarvestVaultInfoTask.java | 4 ++- .../ethparser/web3/EthBlockService.java | 1 + .../belbix/ethparser/web3/SimpleDecoder.java | 33 +++++++++++++++---- .../web3/contracts/UniswapV3Pools.java | 6 +++- .../ethparser/web3/SimpleDecoderTest.java | 23 ++++++++++++- 8 files changed, 70 insertions(+), 22 deletions(-) diff --git a/src/main/java/pro/belbix/ethparser/entity/contracts/VaultEntity.java b/src/main/java/pro/belbix/ethparser/entity/contracts/VaultEntity.java index 3382ece6..f3cecb59 100644 --- a/src/main/java/pro/belbix/ethparser/entity/contracts/VaultEntity.java +++ b/src/main/java/pro/belbix/ethparser/entity/contracts/VaultEntity.java @@ -22,7 +22,7 @@ public class VaultEntity { @Id - @GeneratedValue(strategy = GenerationType.SEQUENCE) + @GeneratedValue(strategy = GenerationType.IDENTITY) private Integer id; @OneToOne(fetch = FetchType.EAGER) @JoinColumn(name = "contract", unique = true) diff --git a/src/main/java/pro/belbix/ethparser/entity/profit/CovalenthqVaultTransactionType.java b/src/main/java/pro/belbix/ethparser/entity/profit/CovalenthqVaultTransactionType.java index 524f67ae..68e04ce9 100644 --- a/src/main/java/pro/belbix/ethparser/entity/profit/CovalenthqVaultTransactionType.java +++ b/src/main/java/pro/belbix/ethparser/entity/profit/CovalenthqVaultTransactionType.java @@ -5,18 +5,20 @@ @FieldDefaults(makeFinal = true, level = AccessLevel.PUBLIC) public enum CovalenthqVaultTransactionType { - WITHDRAW_UNI("account", "writeAmount", "Withdraw"), - WITHDRAW("provider", "value", "Withdraw"), - DEPOSIT_UNI("user", "amount", "Deposit"), - DEPOSIT("dst", "wad", "Deposit"); + WITHDRAW_UNI("account", "writeAmount", "Withdraw", 3), + WITHDRAW("provider", "value", "Withdraw", 2), + DEPOSIT_UNI("user", "amount", "Deposit", 3), + DEPOSIT("dst", "wad", "Deposit", 2); - CovalenthqVaultTransactionType(String address, String value, String type) { + CovalenthqVaultTransactionType(String address, String value, String type, int paramSize) { this.address = address; this.value = value; this.type = type; + this.paramSize = paramSize; } String address; String value; String type; + int paramSize; } diff --git a/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java b/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java index 6fb9b8d2..143326b1 100644 --- a/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java +++ b/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java @@ -85,11 +85,10 @@ public void start() { log.info("Begin parse vault tx"); var executor = Executors.newFixedThreadPool(maxThreadSize); - vaultRepository.findAllByNetwork(ETH_NETWORK) + vaultRepository.findAll() .stream() .map(i -> CompletableFuture.runAsync(() -> getVaultTransaction(i), executor)) .collect(Collectors.toList()); - } private void getVaultTransaction(VaultEntity vault) { @@ -192,7 +191,7 @@ private CovalenthqVaultTransaction toCovalenthqVaultTransaction( log.error("Can not find log event"); throw new IllegalStateException(); }); - var value = getParamValue(item, covalenthqType.value, contractAddress, network); + var value = getParamValue(item, covalenthqType.value, contractAddress, network, covalenthqType.paramSize); var decimal = item.getContractDecimal() == 0 ? functionsUtils.getDecimal(contractAddress, network) : item.getContractDecimal(); @@ -203,7 +202,7 @@ private CovalenthqVaultTransaction toCovalenthqVaultTransaction( transaction.setContractDecimal(decimal); transaction.setContractAddress(contractAddress); transaction.setType(covalenthqType.type); - transaction.setOwnerAddress(getParamValue(item, covalenthqType.address, contractAddress, network)); + transaction.setOwnerAddress(getParamValue(item, covalenthqType.address, contractAddress, network, covalenthqType.paramSize)); transaction.setValue( StringUtils.isEmpty(value) ? BigDecimal.ZERO : new BigDecimal(value) ); @@ -225,12 +224,12 @@ private CovalenthqVaultTransactionType toCovalenthqVaultTransactionType(Covalent return null; } - private String getParamValue(CovalenthqTransactionHistoryItemLog item, String param, String address, String network) { + private String getParamValue(CovalenthqTransactionHistoryItemLog item, String param, String address, String network, int paramSize) { var ethLog = new Log(); ethLog.setTopics(item.getTopics()); ethLog.setData(item.getData()); - List result = simpleDecoder.decodeEthLogForDepositAndWithdraw(ethLog) + List result = simpleDecoder.decodeEthLogForDepositAndWithdraw(ethLog, network, paramSize) .orElseThrow(); if (ethLog.getData() == null && DecodeExcludeConstants.DECODE_ONLY_TOPICS.get(ETH_NETWORK).contains(address.toLowerCase())) { diff --git a/src/main/java/pro/belbix/ethparser/service/task/HarvestVaultInfoTask.java b/src/main/java/pro/belbix/ethparser/service/task/HarvestVaultInfoTask.java index 1f029d08..c69bb82f 100644 --- a/src/main/java/pro/belbix/ethparser/service/task/HarvestVaultInfoTask.java +++ b/src/main/java/pro/belbix/ethparser/service/task/HarvestVaultInfoTask.java @@ -15,6 +15,7 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; import pro.belbix.ethparser.entity.contracts.ContractEntity; import pro.belbix.ethparser.entity.contracts.VaultEntity; import pro.belbix.ethparser.model.HarvestVaultInfo.HarvestVaultItemInfo; @@ -79,7 +80,8 @@ public void start() { } } - private List doTaskByAddressAndNetwork(Map items, String network) { + @Transactional + public List doTaskByAddressAndNetwork(Map items, String network) { log.info("Begin find vault and insert in network: {}", network); var existVaults = contractRepository.findAllByNetworkAndInAddressAndType(network, items.values().stream().map(i -> i.getVaultAddress().toLowerCase()).collect(Collectors.toList()), ContractType.VAULT.getId()); diff --git a/src/main/java/pro/belbix/ethparser/web3/EthBlockService.java b/src/main/java/pro/belbix/ethparser/web3/EthBlockService.java index a6445f08..1e06a9b3 100644 --- a/src/main/java/pro/belbix/ethparser/web3/EthBlockService.java +++ b/src/main/java/pro/belbix/ethparser/web3/EthBlockService.java @@ -31,6 +31,7 @@ public synchronized long getTimestampSecForBlock(long blockNumber, String networ throw new IllegalStateException("Can't fetch block for " + blockNumber); } + // TODO ERROR: duplicate key value violates unique constraint "block_cache_pkey" cachedBlock = new BlockCacheEntity(); cachedBlock.setBlock(blockNumber); cachedBlock.setBlockDate(extractDateFromBlock(block)); diff --git a/src/main/java/pro/belbix/ethparser/web3/SimpleDecoder.java b/src/main/java/pro/belbix/ethparser/web3/SimpleDecoder.java index e1559fc2..073e0d55 100644 --- a/src/main/java/pro/belbix/ethparser/web3/SimpleDecoder.java +++ b/src/main/java/pro/belbix/ethparser/web3/SimpleDecoder.java @@ -1,6 +1,7 @@ package pro.belbix.ethparser.web3; import static org.web3j.abi.FunctionReturnDecoder.decodeIndexedValue; +import static pro.belbix.ethparser.service.AbiProviderService.BSC_NETWORK; import java.util.ArrayList; import java.util.List; @@ -28,11 +29,11 @@ public Optional> decodeEthLog(Log ethLog) { extractLogIndexedValues(ethLog.getTopics(), ethLog.getData(), parameters)); } - public Optional> decodeEthLogForDepositAndWithdraw(Log ethLog) { + public Optional> decodeEthLogForDepositAndWithdraw(Log ethLog, String network, int paramSize) { return parseMethodId(ethLog) .flatMap(this::findParameters) .map(parameters -> - extractLogIndexedValuesWrapped(ethLog.getTopics(), ethLog.getData(), parameters)); + extractLogIndexedValuesWrapped(ethLog.getTopics(), ethLog.getData(), parameters, network, paramSize)); } // deposit and withdraw @@ -48,15 +49,33 @@ public List decodeOnlyTopics(Log ethLog) { } } - public List extractLogIndexedValuesWrapped( - List topics, - String data, - List> parameters) { + // deposit and withdraw + public List decodeOnlyData(List topics, String data, int paramSize) { + try { + return paramSize > 2 ? + extractLogIndexedValues(topics, data, List.of( + TypeReference.makeTypeReference("address", false, false), + TypeReference.makeTypeReference("uint256", false, false), + TypeReference.makeTypeReference("uint256", false, false) + )) : + extractLogIndexedValues(topics, data, List.of( + TypeReference.makeTypeReference("address", false, false), + TypeReference.makeTypeReference("uint256", false, false) + )); + } catch (Exception e) { + log.error("Error during decodeOnlyTopics", e); + return List.of(); + } + } + + public List extractLogIndexedValuesWrapped(List topics, String data, List> parameters, String network, int paramSize) { try { return extractLogIndexedValues(topics, data, parameters); } catch (Exception e) { log.error("Get error during parse log: {}", e.getMessage()); - return extractLogIndexValues(topics, data); + return BSC_NETWORK.equals(network) ? + decodeOnlyData(topics, data, paramSize) : + extractLogIndexValues(topics, data); } } diff --git a/src/main/java/pro/belbix/ethparser/web3/contracts/UniswapV3Pools.java b/src/main/java/pro/belbix/ethparser/web3/contracts/UniswapV3Pools.java index 38bfa3e2..2414c145 100644 --- a/src/main/java/pro/belbix/ethparser/web3/contracts/UniswapV3Pools.java +++ b/src/main/java/pro/belbix/ethparser/web3/contracts/UniswapV3Pools.java @@ -20,6 +20,7 @@ public interface UniswapV3Pools { ); + // if you want to add new value, use contract 0x1f98431c8ad98523631ae4a59f267346ea31f984 getPool Map> VAULTS_TO_POOLS = Map.of( ETH_NETWORK, Map.ofEntries( new AbstractMap.SimpleEntry<>( @@ -119,7 +120,10 @@ public interface UniswapV3Pools { // exclued BUSD_USDC new AbstractMap.SimpleEntry<>( "0x7095b06C02B66e4133F7B4b078B2720CB4437408".toLowerCase(), - "") + ""), + new AbstractMap.SimpleEntry<>( + "0x2CE57694b635f6Ea0087A341654543E12b082538".toLowerCase(), + "0x02DaA5fdd4B474c13A8D6D141471B87FBd2452cd") ), BSC_NETWORK, Map.of(), MATIC_NETWORK, Map.of() diff --git a/src/test/java/pro/belbix/ethparser/web3/SimpleDecoderTest.java b/src/test/java/pro/belbix/ethparser/web3/SimpleDecoderTest.java index 81e7c5ef..68752cbd 100644 --- a/src/test/java/pro/belbix/ethparser/web3/SimpleDecoderTest.java +++ b/src/test/java/pro/belbix/ethparser/web3/SimpleDecoderTest.java @@ -1,6 +1,9 @@ package pro.belbix.ethparser.web3; import static org.assertj.core.api.AssertionsForClassTypes.assertThat; +import static pro.belbix.ethparser.entity.profit.CovalenthqVaultTransactionType.DEPOSIT; +import static pro.belbix.ethparser.entity.profit.CovalenthqVaultTransactionType.DEPOSIT_UNI; +import static pro.belbix.ethparser.service.AbiProviderService.ETH_NETWORK; import java.math.BigInteger; import java.util.List; @@ -21,7 +24,7 @@ public void decodeUniswapV3EthLogTest_deposit() { )); log.setData("0x00000000000000000000000000000000000000000000005053b0b935131f65ee00000000000000000000000000000000000000000000000000000000578f2940"); - var result = decoder.decodeEthLogForDepositAndWithdraw(log).orElseThrow(); + var result = decoder.decodeEthLogForDepositAndWithdraw(log, ETH_NETWORK, DEPOSIT_UNI.paramSize).orElseThrow(); assertThat(result).isNotNull(); assertThat(result.size()).isEqualTo(3); @@ -44,4 +47,22 @@ public void decodeOnlyTopics_deposit() { assertThat(result.size()).isEqualTo(2); assertThat(result.get(1).getValue()).isEqualTo(new BigInteger("28000000000000000000")); } + + @Test + public void decodeOnlyData_deposit() { + var result = decoder.decodeOnlyData(List.of("0x90890809c654f11d6e72a28fa60149770a0d11ec6c92319d6ceb2bb0a4ea1a15"), + "0x000000000000000000000000bb4cdb9cbd36b01bd1cbaebf2de08d9173bc095c00000000000000000000000000000000000000000000000000572ecd9c5cd68d00000000000000000000000000000000000000000000000000572ecd9c5cd68d", + DEPOSIT_UNI.paramSize); + + assertThat(result).isNotNull(); + } + + @Test + public void decodeOnlyData_deposit_withTwoValueData() { + var result = decoder.decodeOnlyData(List.of("0xe1fffcc4923d04b559f4d29a8bfc6cda04eb5b0d3c460751c2402c5c5cc9109c"), + "0x0000000000000000000000000cc0f3edcede8504e63d4fe89cd2dcc1d41ae46f000000000000000000000000000000000000000000000000016345785d8a0000", + DEPOSIT.paramSize); + + assertThat(result).isNotNull(); + } } From 54bd1b9396aab3113c97cbfe255100a3739d3f47 Mon Sep 17 00:00:00 2001 From: alex Date: Fri, 22 Apr 2022 07:54:10 +0300 Subject: [PATCH 48/65] Change logic for handle event for profit and generate id for eth_vaults --- .../entity/contracts/VaultEntity.java | 5 +- .../CovalenthqVaultTransactionType.java | 13 +- .../model/CovalenthqTransactionHistory.java | 2 + .../ethparser/service/ProfitService.java | 58 ++++- .../task/CovalenthqTransactionTask.java | 126 +--------- .../service/task/HarvestVaultInfoTask.java | 2 + .../ethparser/utils/profit/ParseLogUtils.java | 228 ++++++++++++++++++ .../web3/contracts/ContractConstantsV5.java | 1 + 8 files changed, 298 insertions(+), 137 deletions(-) create mode 100644 src/main/java/pro/belbix/ethparser/utils/profit/ParseLogUtils.java diff --git a/src/main/java/pro/belbix/ethparser/entity/contracts/VaultEntity.java b/src/main/java/pro/belbix/ethparser/entity/contracts/VaultEntity.java index f3cecb59..9865a342 100644 --- a/src/main/java/pro/belbix/ethparser/entity/contracts/VaultEntity.java +++ b/src/main/java/pro/belbix/ethparser/entity/contracts/VaultEntity.java @@ -2,8 +2,6 @@ import javax.persistence.Entity; import javax.persistence.FetchType; -import javax.persistence.GeneratedValue; -import javax.persistence.GenerationType; import javax.persistence.Id; import javax.persistence.Index; import javax.persistence.JoinColumn; @@ -22,7 +20,8 @@ public class VaultEntity { @Id - @GeneratedValue(strategy = GenerationType.IDENTITY) +// TODO Can not save for HarvestVaultInfoTask, after return GeneratedValue annotation +// @GeneratedValue(strategy = GenerationType.IDENTITY) private Integer id; @OneToOne(fetch = FetchType.EAGER) @JoinColumn(name = "contract", unique = true) diff --git a/src/main/java/pro/belbix/ethparser/entity/profit/CovalenthqVaultTransactionType.java b/src/main/java/pro/belbix/ethparser/entity/profit/CovalenthqVaultTransactionType.java index 68e04ce9..dcaa49b4 100644 --- a/src/main/java/pro/belbix/ethparser/entity/profit/CovalenthqVaultTransactionType.java +++ b/src/main/java/pro/belbix/ethparser/entity/profit/CovalenthqVaultTransactionType.java @@ -8,16 +8,27 @@ public enum CovalenthqVaultTransactionType { WITHDRAW_UNI("account", "writeAmount", "Withdraw", 3), WITHDRAW("provider", "value", "Withdraw", 2), DEPOSIT_UNI("user", "amount", "Deposit", 3), - DEPOSIT("dst", "wad", "Deposit", 2); + DEPOSIT("dst", "wad", "Deposit", 2), + TRANSFER("from", "to", "valueTransfer", "Transfer", 3); CovalenthqVaultTransactionType(String address, String value, String type, int paramSize) { this.address = address; this.value = value; this.type = type; this.paramSize = paramSize; + this.toAddress = ""; + } + + CovalenthqVaultTransactionType(String address, String toAddress, String value, String type, int paramSize) { + this.address = address; + this.value = value; + this.type = type; + this.paramSize = paramSize; + this.toAddress = toAddress; } String address; + String toAddress; String value; String type; int paramSize; diff --git a/src/main/java/pro/belbix/ethparser/model/CovalenthqTransactionHistory.java b/src/main/java/pro/belbix/ethparser/model/CovalenthqTransactionHistory.java index e0823204..cf654571 100644 --- a/src/main/java/pro/belbix/ethparser/model/CovalenthqTransactionHistory.java +++ b/src/main/java/pro/belbix/ethparser/model/CovalenthqTransactionHistory.java @@ -54,6 +54,8 @@ public static class CovalenthqTransactionHistoryItemLog { private List topics; @JsonProperty("raw_log_data") private String data; + @JsonProperty("sender_address") + private String senderAddress; } } diff --git a/src/main/java/pro/belbix/ethparser/service/ProfitService.java b/src/main/java/pro/belbix/ethparser/service/ProfitService.java index 7de56887..f42e9aad 100644 --- a/src/main/java/pro/belbix/ethparser/service/ProfitService.java +++ b/src/main/java/pro/belbix/ethparser/service/ProfitService.java @@ -6,6 +6,7 @@ import java.math.BigDecimal; import java.math.BigInteger; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -224,18 +225,51 @@ private BigDecimal calculateTxProfit(List transactio } private List calculateProfitByVaults(List transactions, List contracts) { - return contracts.stream() - .map(i -> { - var txByContract = transactions.stream() - .filter(tx -> tx.getContractAddress().equalsIgnoreCase(i.getAddress())) - .collect(Collectors.toList()); - - return ProfitListResultItem.builder() - .contractAddress(i.getAddress()) - .name(i.getName()) - .profit(calculateTxProfit(txByContract)) - .build(); - }) + return joinAnyVersionVaults( + contracts.stream() + .map(i -> { + var txByContract = transactions.stream() + .filter(tx -> tx.getContractAddress().equalsIgnoreCase(i.getAddress())) + .collect(Collectors.toList()); + + return ProfitListResultItem.builder() + .contractAddress(i.getAddress()) + .name(i.getName()) + .profit(calculateTxProfit(txByContract)) + .build(); + }) + .collect(Collectors.toList()) + ); + } + + private List joinAnyVersionVaults(List result) { + result = result.stream() + .sorted((i1, i2) -> i1.getName().compareToIgnoreCase(i2.getName())) .collect(Collectors.toList()); + + var checkedResult = new ArrayList(); + var returnResult = new ArrayList(); + for (ProfitListResultItem item : result) { + if (checkedResult.stream().anyMatch(i -> item.equals(i))) { + continue; + } + + var values = result.stream() + .filter(i -> i.getName().startsWith(item.getName())) + .collect(Collectors.toList()); + + checkedResult.addAll(values); + returnResult.add(ProfitListResultItem.builder() + .profit( + values.stream() + .map(i -> i.getProfit()) + .reduce(BigDecimal.ZERO, BigDecimal::add) + ) + .name(item.getName()) + .contractAddress(item.getContractAddress()) + .build()); + } + + return returnResult; } } diff --git a/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java b/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java index 143326b1..bb100950 100644 --- a/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java +++ b/src/main/java/pro/belbix/ethparser/service/task/CovalenthqTransactionTask.java @@ -1,48 +1,30 @@ package pro.belbix.ethparser.service.task; -import static pro.belbix.ethparser.entity.profit.CovalenthqVaultTransactionType.DEPOSIT; -import static pro.belbix.ethparser.entity.profit.CovalenthqVaultTransactionType.DEPOSIT_UNI; -import static pro.belbix.ethparser.entity.profit.CovalenthqVaultTransactionType.WITHDRAW; -import static pro.belbix.ethparser.entity.profit.CovalenthqVaultTransactionType.WITHDRAW_UNI; -import static pro.belbix.ethparser.service.AbiProviderService.ETH_NETWORK; - -import java.math.BigDecimal; -import java.math.BigInteger; import java.util.Collection; import java.util.HashSet; import java.util.LinkedList; -import java.util.List; -import java.util.Map; import java.util.Objects; -import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executors; import java.util.stream.Collectors; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Sort; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; -import org.web3j.abi.datatypes.Type; -import org.web3j.protocol.core.methods.response.Log; import pro.belbix.ethparser.entity.contracts.VaultEntity; import pro.belbix.ethparser.entity.profit.CovalenthqVaultTransaction; -import pro.belbix.ethparser.entity.profit.CovalenthqVaultTransactionType; import pro.belbix.ethparser.error.exceptions.CanNotFetchPriceException; import pro.belbix.ethparser.model.CovalenthqTransactionHistory.CovalenthqTransactionHistoryItems.CovalenthqTransactionHistoryItem; -import pro.belbix.ethparser.model.CovalenthqTransactionHistory.CovalenthqTransactionHistoryItems.CovalenthqTransactionHistoryItem.CovalenthqTransactionHistoryItemLog; import pro.belbix.ethparser.repositories.covalenthq.CovalenthqVaultTransactionRepository; import pro.belbix.ethparser.repositories.eth.VaultRepository; import pro.belbix.ethparser.service.SharePriceService; import pro.belbix.ethparser.service.TokenPriceService; import pro.belbix.ethparser.service.external.CovalenthqService; -import pro.belbix.ethparser.web3.SimpleDecoder; +import pro.belbix.ethparser.utils.profit.ParseLogUtils; import pro.belbix.ethparser.web3.Web3Functions; -import pro.belbix.ethparser.web3.abi.FunctionsUtils; -import pro.belbix.ethparser.web3.contracts.DecodeExcludeConstants; @Service @RequiredArgsConstructor @@ -53,28 +35,19 @@ public class CovalenthqTransactionTask { private static final int MINUS_BLOCK = 10; private static final int PAGINATION_SIZE = 10000; private static final int PAGINATION_SIZE_FOR_A_LOT_OF_DATA = 100; - private static final Map LOG_EVENTS = Map.of( - "0xf279e6a1f5e320cca91135676d9cb6e44ca8a08c0b88342bcdb1144f6511b568", WITHDRAW_UNI, - "0x884edad9ce6fa2440d8a54cc123490eb96d2768479d49ff9c7366125a9424364", WITHDRAW, - "0x90890809c654f11d6e72a28fa60149770a0d11ec6c92319d6ceb2bb0a4ea1a15", DEPOSIT_UNI, - "0xe1fffcc4923d04b559f4d29a8bfc6cda04eb5b0d3c460751c2402c5c5cc9109c", DEPOSIT - ); - @Value("${task.transaction.enable}") private Boolean enable; @Value("${task.transaction.max-thread-size}") private Integer maxThreadSize; - private final SimpleDecoder simpleDecoder; private final VaultRepository vaultRepository; private final CovalenthqService covalenthqService; private final CovalenthqVaultTransactionRepository covalenthqVaultTransactionRepository; private final Web3Functions web3Functions; private final SharePriceService sharePriceService; private final TokenPriceService tokenPriceService; - private final FunctionsUtils functionsUtils; - + private final ParseLogUtils parseLogUtils; @Scheduled(fixedRateString = "${task.transaction.fixedRate}") public void start() { @@ -141,8 +114,9 @@ private void fetchTransactions(String address, String network, long startingBloc var transactions = result.stream() .map(CovalenthqTransactionHistoryItem::getLogs) .flatMap(Collection::stream) - .filter(this::isDepositOrWithdraw) - .map(log -> toCovalenthqVaultTransaction(log, network, address)) + .filter(i -> parseLogUtils.isCorrectLog(i, address)) + .map(log -> parseLogUtils.toCovalenthqVaultTransactionFromTransfer(log, network, address)) + .flatMap(Collection::stream) .filter(Objects::nonNull) .collect(Collectors.toList()); @@ -178,96 +152,6 @@ private void fetchTransactions(String address, String network, long startingBloc } } - private boolean isDepositOrWithdraw(CovalenthqTransactionHistoryItemLog log) { - return log.getTopics() != null && !log.getTopics().isEmpty() && LOG_EVENTS.get(log.getTopics().get(0)) != null; - } - - private CovalenthqVaultTransaction toCovalenthqVaultTransaction( - CovalenthqTransactionHistoryItemLog item, String network, String contractAddress) { - try { - var transaction = new CovalenthqVaultTransaction(); - var covalenthqType = Optional.ofNullable(toCovalenthqVaultTransactionType(item)) - .orElseThrow(() -> { - log.error("Can not find log event"); - throw new IllegalStateException(); - }); - var value = getParamValue(item, covalenthqType.value, contractAddress, network, covalenthqType.paramSize); - var decimal = item.getContractDecimal() == 0 - ? functionsUtils.getDecimal(contractAddress, network) - : item.getContractDecimal(); - - transaction.setNetwork(network); - transaction.setBlock(item.getBlockHeight()); - transaction.setTransactionHash(item.getTransactionHash()); - transaction.setContractDecimal(decimal); - transaction.setContractAddress(contractAddress); - transaction.setType(covalenthqType.type); - transaction.setOwnerAddress(getParamValue(item, covalenthqType.address, contractAddress, network, covalenthqType.paramSize)); - transaction.setValue( - StringUtils.isEmpty(value) ? BigDecimal.ZERO : new BigDecimal(value) - ); - transaction.setSignedAt(item.getSignedAt()); - - return transaction; - } catch (Exception e) { - log.error("Can not parse covalenthq log: {}", item, e); - throw new CanNotFetchPriceException(); -// return null; - } - } - - private CovalenthqVaultTransactionType toCovalenthqVaultTransactionType(CovalenthqTransactionHistoryItemLog log) { - if (log.getTopics() != null && !log.getTopics().isEmpty()) { - return LOG_EVENTS.get(log.getTopics().get(0)); - } - - return null; - } - - private String getParamValue(CovalenthqTransactionHistoryItemLog item, String param, String address, String network, int paramSize) { - var ethLog = new Log(); - ethLog.setTopics(item.getTopics()); - ethLog.setData(item.getData()); - - List result = simpleDecoder.decodeEthLogForDepositAndWithdraw(ethLog, network, paramSize) - .orElseThrow(); - - if (ethLog.getData() == null && DecodeExcludeConstants.DECODE_ONLY_TOPICS.get(ETH_NETWORK).contains(address.toLowerCase())) { - ethLog.setData(""); - result = simpleDecoder.decodeOnlyTopics(ethLog); - } - - - var indexParam = -1; - var castToString = false; - - switch (param) { - case "account": - case "user": - case "dst": - case "provider": - indexParam = 0; - castToString = true; - break; - case "wad": - case "value": - indexParam = 1; - break; - case "amount" : - case "writeAmount": - indexParam = 2; - break; - } - if (result.isEmpty() || indexParam == -1 || indexParam >= result.size()) { - log.error("Unknown param or can not parse data"); - throw new IllegalStateException(); - } - - return castToString - ? (String) result.get(indexParam).getValue() - : ((BigInteger) result.get(indexParam).getValue()).toString(); - } - private CovalenthqVaultTransaction fillAdditionalParams(CovalenthqVaultTransaction covalenthqVaultTransaction) { var vaultAddress = covalenthqVaultTransaction.getContractAddress(); var block = covalenthqVaultTransaction.getBlock(); diff --git a/src/main/java/pro/belbix/ethparser/service/task/HarvestVaultInfoTask.java b/src/main/java/pro/belbix/ethparser/service/task/HarvestVaultInfoTask.java index c69bb82f..3895be38 100644 --- a/src/main/java/pro/belbix/ethparser/service/task/HarvestVaultInfoTask.java +++ b/src/main/java/pro/belbix/ethparser/service/task/HarvestVaultInfoTask.java @@ -122,6 +122,8 @@ public List doTaskByAddressAndNetwork(Map LOG_EVENTS_V1 = Map.of( + "0xf279e6a1f5e320cca91135676d9cb6e44ca8a08c0b88342bcdb1144f6511b568", WITHDRAW_UNI, + "0x884edad9ce6fa2440d8a54cc123490eb96d2768479d49ff9c7366125a9424364", WITHDRAW, + "0x90890809c654f11d6e72a28fa60149770a0d11ec6c92319d6ceb2bb0a4ea1a15", DEPOSIT_UNI, + "0xe1fffcc4923d04b559f4d29a8bfc6cda04eb5b0d3c460751c2402c5c5cc9109c", DEPOSIT + ); + + private static final Map LOG_EVENTS_V2 = Map.of( + "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef", TRANSFER + ); + + FunctionsUtils functionsUtils; + SimpleDecoder simpleDecoder; + + + // filter deposit and withdraw + // filter by contractAddress + public boolean isCorrectLog(CovalenthqTransactionHistoryItemLog log, String contractAddress) { + return log.getTopics() != null + && !log.getTopics().isEmpty() + && log.getSenderAddress().equalsIgnoreCase(contractAddress) + && LOG_EVENTS_V2.get(log.getTopics().get(0)) != null; + } + + // only for event logs deposit and withdraw + public CovalenthqVaultTransaction toCovalenthqVaultTransactionFromDepositOrWithdraw(CovalenthqTransactionHistoryItemLog item, + String network, String contractAddress) { + try { + var transaction = new CovalenthqVaultTransaction(); + var covalenthqType = Optional.ofNullable(toCovalenthqVaultTransactionTypeFromDepositOrWithdraw(item)) + .orElseThrow(() -> { + log.error("Can not find log event name"); + throw new IllegalStateException(); + }); + var value = getParamValue(item, covalenthqType.value, contractAddress, network, covalenthqType.paramSize); + var decimal = item.getContractDecimal() == 0 + ? functionsUtils.getDecimal(contractAddress, network) + : item.getContractDecimal(); + + transaction.setNetwork(network); + transaction.setBlock(item.getBlockHeight()); + transaction.setTransactionHash(item.getTransactionHash()); + transaction.setContractDecimal(decimal); + transaction.setContractAddress(contractAddress); + transaction.setType(covalenthqType.type); + transaction.setOwnerAddress(getParamValue(item, covalenthqType.address, contractAddress, network, covalenthqType.paramSize)); + transaction.setValue( + StringUtils.isEmpty(value) ? BigDecimal.ZERO : new BigDecimal(value) + ); + transaction.setSignedAt(item.getSignedAt()); + + return transaction; + } catch (Exception e) { + log.error("Can not parse covalenthq log: {}", item, e); + throw new CanNotFetchPriceException(); + } + } + + // only for event logs transfer + public List toCovalenthqVaultTransactionFromTransfer(CovalenthqTransactionHistoryItemLog item, + String network, String contractAddress) { + try { + var covalenthqType = Optional.ofNullable(toCovalenthqVaultTransactionTypeFromTransfer(item)) + .orElseThrow(() -> { + log.error("Can not find log event name"); + throw new IllegalStateException(); + }); + var value = getParamValue(item, covalenthqType.value); + var fromAddress = getParamValue(item, covalenthqType.address); + var toAddress = getParamValue(item, covalenthqType.toAddress); + + var transactions = new ArrayList(); + + if (StringUtils.isNotEmpty(fromAddress) && !ZERO_ADDRESS.equalsIgnoreCase(fromAddress)) { + transactions.add(toCovalenthqVaultTransaction(item, contractAddress, network, WITHDRAW.type, fromAddress, value)); + } + + if (StringUtils.isNotEmpty(toAddress) && !ZERO_ADDRESS.equalsIgnoreCase(toAddress)) { + transactions.add(toCovalenthqVaultTransaction(item, contractAddress, network, DEPOSIT.type, toAddress, value)); + } + + return transactions; + } catch (Exception e) { + log.error("Can not parse covalenthq log: {}", item, e); + throw new CanNotFetchPriceException(); + } + } + + private CovalenthqVaultTransaction toCovalenthqVaultTransaction(CovalenthqTransactionHistoryItemLog item, String contractAddress, + String network, String type, String ownerAddress, String value) { + var transaction = new CovalenthqVaultTransaction(); + var decimal = item.getContractDecimal() == 0 + ? functionsUtils.getDecimal(contractAddress, network) + : item.getContractDecimal(); + + transaction.setNetwork(network); + transaction.setBlock(item.getBlockHeight()); + transaction.setTransactionHash(item.getTransactionHash()); + transaction.setContractDecimal(decimal); + transaction.setContractAddress(contractAddress); + transaction.setType(type); + transaction.setOwnerAddress(ownerAddress); + transaction.setValue( + StringUtils.isEmpty(value) ? BigDecimal.ZERO : new BigDecimal(value) + ); + transaction.setSignedAt(item.getSignedAt()); + + return transaction; + } + + private CovalenthqVaultTransactionType toCovalenthqVaultTransactionTypeFromDepositOrWithdraw(CovalenthqTransactionHistoryItemLog log) { + if (log.getTopics() != null && !log.getTopics().isEmpty()) { + return LOG_EVENTS_V1.get(log.getTopics().get(0)); + } + + return null; + } + + private CovalenthqVaultTransactionType toCovalenthqVaultTransactionTypeFromTransfer(CovalenthqTransactionHistoryItemLog log) { + if (log.getTopics() != null && !log.getTopics().isEmpty()) { + return LOG_EVENTS_V2.get(log.getTopics().get(0)); + } + + return null; + } + + private String getParamValue(CovalenthqTransactionHistoryItemLog item, String param, String address, String network, int paramSize) { + var ethLog = new Log(); + ethLog.setTopics(item.getTopics()); + ethLog.setData(item.getData()); + + List result = simpleDecoder.decodeEthLogForDepositAndWithdraw(ethLog, network, paramSize) + .orElseThrow(); + + if (ethLog.getData() == null && DecodeExcludeConstants.DECODE_ONLY_TOPICS.get(ETH_NETWORK).contains(address.toLowerCase())) { + ethLog.setData(StringUtils.EMPTY); + result = simpleDecoder.decodeOnlyTopics(ethLog); + } + + return getParamValue(param, result); + } + + private String getParamValue(CovalenthqTransactionHistoryItemLog item, String param) { + var ethLog = new Log(); + ethLog.setTopics(item.getTopics()); + ethLog.setData(item.getData()); + + List result = simpleDecoder.decodeEthLog(ethLog) + .orElseThrow(); + + return getParamValue(param, result); + } + + private String getParamValue(String param, List result) { + + var indexParam = -1; + var castToString = false; + + switch (param) { + case "account": + case "user": + case "dst": + case "provider": + case "from": + indexParam = 0; + castToString = true; + break; + case "wad": + case "value": + indexParam = 1; + break; + case "to": + indexParam = 1; + castToString = true; + break; + case "amount" : + case "valueTransfer": + case "writeAmount": + indexParam = 2; + break; + } + if (result.isEmpty() || indexParam == -1 || indexParam >= result.size()) { + log.error("Unknown param or can not parse data"); + throw new IllegalStateException(); + } + + return castToString + ? (String) result.get(indexParam).getValue() + : ((BigInteger) result.get(indexParam).getValue()).toString(); + } +} diff --git a/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstantsV5.java b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstantsV5.java index 2936a909..b1c53a8a 100644 --- a/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstantsV5.java +++ b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstantsV5.java @@ -8,6 +8,7 @@ public interface ContractConstantsV5 { "0xe9e7cea3dedca5984780bafc599bd69add087d56".toLowerCase(), //BUSD "0xdAC17F958D2ee523a2206206994597C13D831ec7".toLowerCase(), //USDT "0x0000000000085d4780B73119b644AE5ecd22b376".toLowerCase(), //TUSD + "0x6B175474E89094C44Da98b954EedeAC495271d0F".toLowerCase(), //DAI "0x2791bca1f2de4661ed88a30c99a7a9449aa84174".toLowerCase(), //matic USDC "0xc2132d05d31c914a87c6611c10748aeb04b58e8f".toLowerCase(), //matic USDT "0xE840B73E5287865EEc17d250bFb1536704B43B21".toLowerCase(), //matic mUSD From a3073e1e320daa48b1f0c61d1e12e764f450999d Mon Sep 17 00:00:00 2001 From: alex Date: Sun, 24 Apr 2022 11:26:25 +0300 Subject: [PATCH 49/65] Add logs for result from harvest api --- .../pro/belbix/ethparser/service/external/HarvestService.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/pro/belbix/ethparser/service/external/HarvestService.java b/src/main/java/pro/belbix/ethparser/service/external/HarvestService.java index 46e1dbfa..a31f2b30 100644 --- a/src/main/java/pro/belbix/ethparser/service/external/HarvestService.java +++ b/src/main/java/pro/belbix/ethparser/service/external/HarvestService.java @@ -28,6 +28,8 @@ public HarvestPoolInfo getPools() { public HarvestVaultInfo getVaults() { var url = String.format(HarvestUrl.VAULTS, externalProperties.getHarvest().getUrl(), externalProperties.getHarvest().getKey()); log.info("Starting get vaults from harvest {} ", url); - return restTemplate.getForObject(url, HarvestVaultInfo.class); + var result = restTemplate.getForEntity(url, HarvestVaultInfo.class); + log.info("Result getting vaults from harvest API : {}", result); + return result.getBody(); } } From 26f8d4d3102db8076c70683ef3dfeb3a942c375e Mon Sep 17 00:00:00 2001 From: alex Date: Sun, 24 Apr 2022 22:31:25 +0300 Subject: [PATCH 50/65] Add test request --- .../pro/belbix/ethparser/service/external/HarvestService.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/pro/belbix/ethparser/service/external/HarvestService.java b/src/main/java/pro/belbix/ethparser/service/external/HarvestService.java index a31f2b30..815b4a0c 100644 --- a/src/main/java/pro/belbix/ethparser/service/external/HarvestService.java +++ b/src/main/java/pro/belbix/ethparser/service/external/HarvestService.java @@ -28,6 +28,8 @@ public HarvestPoolInfo getPools() { public HarvestVaultInfo getVaults() { var url = String.format(HarvestUrl.VAULTS, externalProperties.getHarvest().getUrl(), externalProperties.getHarvest().getKey()); log.info("Starting get vaults from harvest {} ", url); + var test = restTemplate.getForEntity("https://httpbin.org/get", String.class); + log.info("Test request: {}", test); var result = restTemplate.getForEntity(url, HarvestVaultInfo.class); log.info("Result getting vaults from harvest API : {}", result); return result.getBody(); From 1d467c67e167ebdad455ff34f053e5d1f5f793f0 Mon Sep 17 00:00:00 2001 From: alex Date: Sun, 24 Apr 2022 23:06:43 +0300 Subject: [PATCH 51/65] Remove test request and fix error with NoClassDefFoundError --- .../belbix/ethparser/service/external/HarvestService.java | 2 -- .../pro/belbix/ethparser/web3/contracts/ContractUtils.java | 6 +++--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/main/java/pro/belbix/ethparser/service/external/HarvestService.java b/src/main/java/pro/belbix/ethparser/service/external/HarvestService.java index 815b4a0c..a31f2b30 100644 --- a/src/main/java/pro/belbix/ethparser/service/external/HarvestService.java +++ b/src/main/java/pro/belbix/ethparser/service/external/HarvestService.java @@ -28,8 +28,6 @@ public HarvestPoolInfo getPools() { public HarvestVaultInfo getVaults() { var url = String.format(HarvestUrl.VAULTS, externalProperties.getHarvest().getUrl(), externalProperties.getHarvest().getKey()); log.info("Starting get vaults from harvest {} ", url); - var test = restTemplate.getForEntity("https://httpbin.org/get", String.class); - log.info("Test request: {}", test); var result = restTemplate.getForEntity(url, HarvestVaultInfo.class); log.info("Result getting vaults from harvest API : {}", result); return result.getBody(); diff --git a/src/main/java/pro/belbix/ethparser/web3/contracts/ContractUtils.java b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractUtils.java index e945e9dc..c99693c8 100644 --- a/src/main/java/pro/belbix/ethparser/web3/contracts/ContractUtils.java +++ b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractUtils.java @@ -58,9 +58,9 @@ public static boolean isPsAddress(String address, String network) { } public static boolean isFarmAddress(String address) { - return FARM_TOKEN.equalsIgnoreCase(address) - || BSC_FARM_TOKEN.equalsIgnoreCase(address) - || MATIC_FARM_TOKEN.equalsIgnoreCase(address); + return "0xa0246c9032bc3a600820415ae600c6388619a14d".equalsIgnoreCase(address) + || "0x4B5C23cac08a567ecf0c1fFcA8372A45a5D33743".equalsIgnoreCase(address) + || "0xab0b2ddb9c7e440fac8e140a89c0dbcbf2d7bbff".equalsIgnoreCase(address); } public static boolean isStableCoin(String address) { From fc87a3a781dcb3d32ec2fc777fc6cc231acf9eaf Mon Sep 17 00:00:00 2001 From: alex Date: Tue, 26 Apr 2022 10:12:32 +0300 Subject: [PATCH 52/65] Try to find problem with NoClassDefFoundError --- .../web3/contracts/ContractConstants.java | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstants.java b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstants.java index e2b51582..ad7e1438 100644 --- a/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstants.java +++ b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstants.java @@ -61,8 +61,18 @@ public class ContractConstants { .toLowerCase(); public static final String PS_V0_ADDRESS = "0x59258F4e15A5fC74A7284055A8094F58108dbD4f" .toLowerCase(); - public static final String iPS_ADDRESS = "0x1571eD0bed4D987fe2b498DdBaE7DFA19519F651" - .toLowerCase(); + public static final String iPS_ADDRESS; + + static { + try { + iPS_ADDRESS = "0x1571eD0bed4D987fe2b498DdBaE7DFA19519F651" + .toLowerCase(); + } catch (Exception e) { + System.out.println("Get error: " + e); + throw e; + } + } + public static final String ST_PS_ADDRESS = "0x8f5adC58b32D4e5Ca02EAC0E293D35855999436C" .toLowerCase(); From 3fdbdf05801050485a3db8cb701a0694b854da24 Mon Sep 17 00:00:00 2001 From: alex Date: Tue, 26 Apr 2022 10:20:00 +0300 Subject: [PATCH 53/65] Try to find problem with NoClassDefFoundError --- .../web3/contracts/ContractConstants.java | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstants.java b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstants.java index ad7e1438..e2b51582 100644 --- a/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstants.java +++ b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstants.java @@ -61,18 +61,8 @@ public class ContractConstants { .toLowerCase(); public static final String PS_V0_ADDRESS = "0x59258F4e15A5fC74A7284055A8094F58108dbD4f" .toLowerCase(); - public static final String iPS_ADDRESS; - - static { - try { - iPS_ADDRESS = "0x1571eD0bed4D987fe2b498DdBaE7DFA19519F651" - .toLowerCase(); - } catch (Exception e) { - System.out.println("Get error: " + e); - throw e; - } - } - + public static final String iPS_ADDRESS = "0x1571eD0bed4D987fe2b498DdBaE7DFA19519F651" + .toLowerCase(); public static final String ST_PS_ADDRESS = "0x8f5adC58b32D4e5Ca02EAC0E293D35855999436C" .toLowerCase(); From 2c20d76dd150d823427e67044922e1e5c096fdd8 Mon Sep 17 00:00:00 2001 From: alex Date: Tue, 26 Apr 2022 23:13:16 +0300 Subject: [PATCH 54/65] Try to find problem with NoClassDefFoundError --- pom.xml | 1 + .../web3/contracts/ContractConstants.java | 26 ------------------- 2 files changed, 1 insertion(+), 26 deletions(-) diff --git a/pom.xml b/pom.xml index 650977bf..0e34783a 100644 --- a/pom.xml +++ b/pom.xml @@ -16,6 +16,7 @@ + 11 11 11 4.8.4 diff --git a/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstants.java b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstants.java index e2b51582..45adf5a6 100644 --- a/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstants.java +++ b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstants.java @@ -239,30 +239,4 @@ public class ContractConstants { MATIC_FARM_TOKEN ) ); - - public static final Map> EXCLUDE_ADDRESSES_FOR_PRICE_SHARE_BY_NETWORK = Map.of( - ETH_NETWORK, List.of( - PS_ADDRESS, - "0x59258f4e15a5fc74a7284055a8094f58108dbd4f".toLowerCase() // POOL - ), - BSC_NETWORK, List.of(), - MATIC_NETWORK, List.of() - ); - - public static final Map> EXCLUDE_JARVIS_STABLECOIN = Map.of( - ETH_NETWORK, List.of(), - BSC_NETWORK, List.of(), - MATIC_NETWORK, List.of( - "0x8ca194A3b22077359b5732DE53373D4afC11DeE3".toLowerCase(), // jCAD - "0x8343091F2499FD4b6174A46D067A920a3b851FF9".toLowerCase() // jJPY - ) - ); - - public static final Map> IS_NOT_AVAILABLE_IN_ORACLE = Map.of( - ETH_NETWORK, List.of(), - BSC_NETWORK, List.of(), - MATIC_NETWORK, List.of( - "0x32d8513eDDa5AEf930080F15270984A043933A95".toLowerCase() // KyberDMM LP jCAD+CADC-f-QUI-MAR22 - ) - ); } From 28481a0f1bc68902bdd057197505b4e126c2efb3 Mon Sep 17 00:00:00 2001 From: alex Date: Tue, 26 Apr 2022 23:18:08 +0300 Subject: [PATCH 55/65] Update maven dependency plugin to 3.0.2 --- pom.xml | 3 +-- src/main/java/pro/belbix/ethparser/service/ProfitService.java | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 0e34783a..0315ab49 100644 --- a/pom.xml +++ b/pom.xml @@ -16,7 +16,6 @@ - 11 11 11 4.8.4 @@ -289,7 +288,7 @@ org.apache.maven.plugins maven-dependency-plugin - 3.0.0 + 3.0.2 copy-dependencies diff --git a/src/main/java/pro/belbix/ethparser/service/ProfitService.java b/src/main/java/pro/belbix/ethparser/service/ProfitService.java index f42e9aad..e93936ec 100644 --- a/src/main/java/pro/belbix/ethparser/service/ProfitService.java +++ b/src/main/java/pro/belbix/ethparser/service/ProfitService.java @@ -238,6 +238,7 @@ private List calculateProfitByVaults(List i.getProfit().compareTo(BigDecimal.ZERO) != 0) .collect(Collectors.toList()) ); } From 1abde35931be2c9b161ef8665931f0dd0d43c8ee Mon Sep 17 00:00:00 2001 From: alex Date: Tue, 26 Apr 2022 23:27:45 +0300 Subject: [PATCH 56/65] Try to resolve problem with NoClassDefFoundError --- scripts/run_app/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/run_app/run.sh b/scripts/run_app/run.sh index 911443ae..c98acf9b 100755 --- a/scripts/run_app/run.sh +++ b/scripts/run_app/run.sh @@ -1,7 +1,7 @@ rm -rf ./lib rm -rf ./logs rm ethparser.jar -mvn install -Dmaven.test.skip=true -f ./../../pom.xml +mvn install -Dmaven.test.skip=true -Dmaven.version=1.0.1 -f ./../../pom.xml cp -R ./../../dist/lib/. ./lib cp ./../../dist/ethparser.jar ./ethparser.jar java -Xmx4g -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=5005 -Dspring.config.location=./../application.yml,run_config.yml -cp ethparser.jar pro.belbix.ethparser.Application From 9ba85402ec18df181b44a40f0812cb9e3390d06b Mon Sep 17 00:00:00 2001 From: alex Date: Wed, 27 Apr 2022 00:02:31 +0300 Subject: [PATCH 57/65] Try to resolve problem with NoClassDefFoundError --- .../pro/belbix/ethparser/utils/MockUtils.java | 2 +- .../recalculation/TransfersRecalculate.java | 2 +- .../web3/bancor/BancorLogDecoder.java | 2 +- .../web3/bancor/BancorPriceParser.java | 2 +- .../web3/contracts/ContractConstants.java | 33 ------------- .../web3/contracts/ContractConstantsV7.java | 12 +++++ .../web3/contracts/ContractUtils.java | 2 +- .../web3/contracts/EthTokenAddresses.java | 2 +- .../web3/contracts/EthVaultAddresses.java | 3 +- .../web3/contracts/db/ContractDbService.java | 4 +- .../deployer/decoder/DeployerDecoder.java | 2 +- .../web3/deployer/transform/UniqueTypes.java | 2 +- .../harvest/db/VaultActionsDBService.java | 4 +- .../harvest/parser/VaultActionsParser.java | 7 ++- .../DeployerTransactionsParserTest.java | 7 +-- .../web3/erc20/parser/TransferParserTest.java | 46 +++++++++---------- .../parser/ImportantEventsParserTest.java | 4 +- .../vault/VaultActionsParserEthTest.java | 2 +- .../detector/EthContractDetectorTest.java | 2 +- 19 files changed, 57 insertions(+), 83 deletions(-) diff --git a/src/main/java/pro/belbix/ethparser/utils/MockUtils.java b/src/main/java/pro/belbix/ethparser/utils/MockUtils.java index 589babd8..6d56066a 100644 --- a/src/main/java/pro/belbix/ethparser/utils/MockUtils.java +++ b/src/main/java/pro/belbix/ethparser/utils/MockUtils.java @@ -6,7 +6,7 @@ import static pro.belbix.ethparser.model.tx.UniswapTx.REMOVE_LIQ; import static pro.belbix.ethparser.service.AbiProviderService.ETH_NETWORK; import static pro.belbix.ethparser.web3.contracts.ContractConstants.BANCOR_CONVERSION_ADDRESS; -import static pro.belbix.ethparser.web3.contracts.ContractConstants.FARM_TOKEN; +import static pro.belbix.ethparser.web3.contracts.ContractConstantsV7.FARM_TOKEN; import java.math.BigInteger; import java.time.Instant; diff --git a/src/main/java/pro/belbix/ethparser/utils/recalculation/TransfersRecalculate.java b/src/main/java/pro/belbix/ethparser/utils/recalculation/TransfersRecalculate.java index b9a3d75d..c81bae8a 100644 --- a/src/main/java/pro/belbix/ethparser/utils/recalculation/TransfersRecalculate.java +++ b/src/main/java/pro/belbix/ethparser/utils/recalculation/TransfersRecalculate.java @@ -1,6 +1,6 @@ package pro.belbix.ethparser.utils.recalculation; -import static pro.belbix.ethparser.web3.contracts.ContractConstants.FARM_TOKEN; +import static pro.belbix.ethparser.web3.contracts.ContractConstantsV7.FARM_TOKEN; import java.util.ArrayList; import java.util.List; diff --git a/src/main/java/pro/belbix/ethparser/web3/bancor/BancorLogDecoder.java b/src/main/java/pro/belbix/ethparser/web3/bancor/BancorLogDecoder.java index ee88cca4..746dd453 100644 --- a/src/main/java/pro/belbix/ethparser/web3/bancor/BancorLogDecoder.java +++ b/src/main/java/pro/belbix/ethparser/web3/bancor/BancorLogDecoder.java @@ -3,7 +3,7 @@ import static pro.belbix.ethparser.model.tx.BancorPriceTx.BNT; import static pro.belbix.ethparser.model.tx.BancorPriceTx.FARM; import static pro.belbix.ethparser.web3.contracts.ContractConstants.BANCOR_CONVERSION_ADDRESS; -import static pro.belbix.ethparser.web3.contracts.ContractConstants.FARM_TOKEN; +import static pro.belbix.ethparser.web3.contracts.ContractConstantsV7.FARM_TOKEN; import java.math.BigInteger; import java.util.List; diff --git a/src/main/java/pro/belbix/ethparser/web3/bancor/BancorPriceParser.java b/src/main/java/pro/belbix/ethparser/web3/bancor/BancorPriceParser.java index a41f9b8b..2c52ebaa 100644 --- a/src/main/java/pro/belbix/ethparser/web3/bancor/BancorPriceParser.java +++ b/src/main/java/pro/belbix/ethparser/web3/bancor/BancorPriceParser.java @@ -5,7 +5,7 @@ import static pro.belbix.ethparser.web3.contracts.ContractConstants.BANCOR_CONVERSION_ADDRESS; import static pro.belbix.ethparser.web3.contracts.ContractConstants.BANCOR_USDC_CONVERT_PATH; import static pro.belbix.ethparser.web3.contracts.ContractConstants.D6; -import static pro.belbix.ethparser.web3.contracts.ContractConstants.FARM_TOKEN; +import static pro.belbix.ethparser.web3.contracts.ContractConstantsV7.FARM_TOKEN; import static pro.belbix.ethparser.web3.contracts.ContractConstants.L18; import java.math.BigDecimal; diff --git a/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstants.java b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstants.java index 45adf5a6..f18e52bc 100644 --- a/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstants.java +++ b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstants.java @@ -49,8 +49,6 @@ public class ContractConstants { .toLowerCase(); public static final String ZERO_ADDRESS = "0x0000000000000000000000000000000000000000"; public static final String CURVE_ZERO_ADDRESS = "0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee"; - public static final String FARM_TOKEN = "0xa0246c9032bc3a600820415ae600c6388619a14d" - .toLowerCase(); public static final String BSC_FARM_TOKEN = "0x4B5C23cac08a567ecf0c1fFcA8372A45a5D33743" .toLowerCase(); public static final String MATIC_FARM_TOKEN = "0xab0b2ddb9c7e440fac8e140a89c0dbcbf2d7bbff" @@ -61,8 +59,6 @@ public class ContractConstants { .toLowerCase(); public static final String PS_V0_ADDRESS = "0x59258F4e15A5fC74A7284055A8094F58108dbD4f" .toLowerCase(); - public static final String iPS_ADDRESS = "0x1571eD0bed4D987fe2b498DdBaE7DFA19519F651" - .toLowerCase(); public static final String ST_PS_ADDRESS = "0x8f5adC58b32D4e5Ca02EAC0E293D35855999436C" .toLowerCase(); @@ -72,12 +68,6 @@ public class ContractConstants { .toLowerCase(); public static final String BANCOR_CONVERSION_ADDRESS = "0x2f9ec37d6ccfff1cab21733bdadede11c823ccb0"; - public final static Map DEPLOYERS = Map.of( - ETH_NETWORK, "0xf00dD244228F51547f0563e60bCa65a30FBF5f7f".toLowerCase(), - BSC_NETWORK, "0xf00dd244228f51547f0563e60bca65a30fbf5f7f".toLowerCase(), - MATIC_NETWORK, "0xf00dd244228f51547f0563e60bca65a30fbf5f7f".toLowerCase() - ); - final static Map> CONTROLLERS = Map.of( ETH_NETWORK, Map.of( 10770087L, "0x222412af183BCeAdEFd72e4Cb1b71f1889953b1C".toLowerCase(), @@ -145,13 +135,6 @@ public class ContractConstants { MATIC_NETWORK, Map.of() ); - public static final Set PS_ADDRESSES = Set.of( - "0x8f5adC58b32D4e5Ca02EAC0E293D35855999436C".toLowerCase(), // ST_PS - FARM_TOKEN, // FARM TOKEN - PS_ADDRESS, // PS - PS_V0_ADDRESS // PS_V0 - ); - // TODO separate by networks public static final Set ONE_DOLLAR_TOKENS = Set.of( "0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48".toLowerCase(), //USDC @@ -223,20 +206,4 @@ public class ContractConstants { SUSHI_FACTORY_ADDRESS, 11333218 ) ); - - public static final Map> PS_ADDRESSES_BY_NETWORK = Map.of( - ETH_NETWORK, List.of( - "0xd3093e3efbe00f010e8f5efe3f1cb5d9b7fe0eb1".toLowerCase(), // - "0x8f5adC58b32D4e5Ca02EAC0E293D35855999436C".toLowerCase(), // ST_PS - FARM_TOKEN, // FARM TOKEN - PS_ADDRESS, // PS - PS_V0_ADDRESS // PS_V0 - ), - BSC_NETWORK, List.of( - BSC_FARM_TOKEN - ), - MATIC_NETWORK, List.of( - MATIC_FARM_TOKEN - ) - ); } diff --git a/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstantsV7.java b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstantsV7.java index 6cec2f48..30304eae 100644 --- a/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstantsV7.java +++ b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstantsV7.java @@ -1,12 +1,18 @@ package pro.belbix.ethparser.web3.contracts; import static pro.belbix.ethparser.service.AbiProviderService.BSC_NETWORK; +import static pro.belbix.ethparser.service.AbiProviderService.ETH_NETWORK; import static pro.belbix.ethparser.service.AbiProviderService.MATIC_NETWORK; import java.util.Map; import pro.belbix.ethparser.model.TokenInfo; public interface ContractConstantsV7 { + String iPS_ADDRESS = "0x1571eD0bed4D987fe2b498DdBaE7DFA19519F651" + .toLowerCase(); + String FARM_TOKEN = "0xa0246c9032bc3a600820415ae600c6388619a14d" + .toLowerCase(); + Map COIN_PRICE_IN_OTHER_CHAIN = Map.of( // Denarius (DEN-0121) to Denarius BSC TokenInfo.builder() @@ -45,4 +51,10 @@ public interface ContractConstantsV7 { .network(BSC_NETWORK) .build() ); + + Map DEPLOYERS = Map.of( + ETH_NETWORK, "0xf00dD244228F51547f0563e60bCa65a30FBF5f7f".toLowerCase(), + BSC_NETWORK, "0xf00dd244228f51547f0563e60bca65a30fbf5f7f".toLowerCase(), + MATIC_NETWORK, "0xf00dd244228f51547f0563e60bca65a30fbf5f7f".toLowerCase() + ); } diff --git a/src/main/java/pro/belbix/ethparser/web3/contracts/ContractUtils.java b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractUtils.java index c99693c8..520a6dd7 100644 --- a/src/main/java/pro/belbix/ethparser/web3/contracts/ContractUtils.java +++ b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractUtils.java @@ -7,7 +7,7 @@ import static pro.belbix.ethparser.web3.contracts.ContractConstants.BSC_FARM_TOKEN; import static pro.belbix.ethparser.web3.contracts.ContractConstants.CONTROLLERS; import static pro.belbix.ethparser.web3.contracts.ContractConstants.ETH_BLOCK_NUMBER_30_AUGUST_2020; -import static pro.belbix.ethparser.web3.contracts.ContractConstants.FARM_TOKEN; +import static pro.belbix.ethparser.web3.contracts.ContractConstantsV7.FARM_TOKEN; import static pro.belbix.ethparser.web3.contracts.ContractConstants.FULL_PARSABLE_UNI_PAIRS; import static pro.belbix.ethparser.web3.contracts.ContractConstants.MATIC_BLOCK_NUMBER_06_JUL_2021; import static pro.belbix.ethparser.web3.contracts.ContractConstants.MATIC_FARM_TOKEN; diff --git a/src/main/java/pro/belbix/ethparser/web3/contracts/EthTokenAddresses.java b/src/main/java/pro/belbix/ethparser/web3/contracts/EthTokenAddresses.java index 699a2b3c..fa58f6ea 100644 --- a/src/main/java/pro/belbix/ethparser/web3/contracts/EthTokenAddresses.java +++ b/src/main/java/pro/belbix/ethparser/web3/contracts/EthTokenAddresses.java @@ -1,6 +1,6 @@ package pro.belbix.ethparser.web3.contracts; -import static pro.belbix.ethparser.web3.contracts.ContractConstants.FARM_TOKEN; +import static pro.belbix.ethparser.web3.contracts.ContractConstantsV7.FARM_TOKEN; import static pro.belbix.ethparser.web3.contracts.ContractConstants.ZERO_ADDRESS; import static pro.belbix.ethparser.web3.contracts.models.TokenContract.createTokenContracts; diff --git a/src/main/java/pro/belbix/ethparser/web3/contracts/EthVaultAddresses.java b/src/main/java/pro/belbix/ethparser/web3/contracts/EthVaultAddresses.java index 2b09b83b..c22c5c7c 100644 --- a/src/main/java/pro/belbix/ethparser/web3/contracts/EthVaultAddresses.java +++ b/src/main/java/pro/belbix/ethparser/web3/contracts/EthVaultAddresses.java @@ -2,7 +2,6 @@ import static pro.belbix.ethparser.web3.contracts.ContractConstants.PS_ADDRESS; import static pro.belbix.ethparser.web3.contracts.ContractConstants.PS_V0_ADDRESS; -import static pro.belbix.ethparser.web3.contracts.ContractConstants.iPS_ADDRESS; import static pro.belbix.ethparser.web3.contracts.models.SimpleContract.createContracts; import java.util.List; @@ -80,7 +79,7 @@ private EthVaultAddresses() { new SimpleContract(11745394, "CRV_GUSD", "0xB8671E33fcFC7FEA2F7a3Ea4a117F065ec4b009E"), new SimpleContract(11830928, "CRV_AAVE", "0xc3EF8C4043D7cf1D15B6bb4cd307C844E0BA9d42"), new SimpleContract(11777480, "SUSHI_SUSHI_ETH", "0x5aDe382F38A09A1F8759D06fFE2067992ab5c78e"), - new SimpleContract(11775913, "iPS", iPS_ADDRESS), + new SimpleContract(11775913, "iPS", ContractConstantsV7.iPS_ADDRESS), new SimpleContract(11905238, "ONEINCH_ETH_ONEINCH", "0xFCA949E34ecd9dE519542CF02054DE707Cf361cE"), new SimpleContract(11924821, "UNI_WBTC_KLON", "0xB4E3fC276532f27Bd0F738928Ce083A3b064ba61"), new SimpleContract(11924877, "UNI_WBTC_KBTC", "0x5cd9Db40639013A08d797A839C9BECD6EC5DCD4D"), diff --git a/src/main/java/pro/belbix/ethparser/web3/contracts/db/ContractDbService.java b/src/main/java/pro/belbix/ethparser/web3/contracts/db/ContractDbService.java index 98475f4d..d22fcc72 100644 --- a/src/main/java/pro/belbix/ethparser/web3/contracts/db/ContractDbService.java +++ b/src/main/java/pro/belbix/ethparser/web3/contracts/db/ContractDbService.java @@ -26,7 +26,7 @@ import pro.belbix.ethparser.repositories.eth.TokenToUniPairRepository; import pro.belbix.ethparser.repositories.eth.UniPairRepository; import pro.belbix.ethparser.repositories.eth.VaultRepository; -import pro.belbix.ethparser.web3.contracts.ContractConstants; +import pro.belbix.ethparser.web3.contracts.ContractConstantsV7; import pro.belbix.ethparser.web3.contracts.ContractType; import pro.belbix.ethparser.web3.contracts.ContractUtils; @@ -241,7 +241,7 @@ public Optional getTokenByAddress(String address, String network) { public List getSubscriptions() { Set contracts = new HashSet<>(Set.of( - ContractConstants.FARM_TOKEN + ContractConstantsV7.FARM_TOKEN )); contracts.addAll(Objects.requireNonNull(getControllerAddressByNetwork(ETH_NETWORK))); contracts.addAll( diff --git a/src/main/java/pro/belbix/ethparser/web3/deployer/decoder/DeployerDecoder.java b/src/main/java/pro/belbix/ethparser/web3/deployer/decoder/DeployerDecoder.java index 9440ed72..6664331d 100644 --- a/src/main/java/pro/belbix/ethparser/web3/deployer/decoder/DeployerDecoder.java +++ b/src/main/java/pro/belbix/ethparser/web3/deployer/decoder/DeployerDecoder.java @@ -1,6 +1,6 @@ package pro.belbix.ethparser.web3.deployer.decoder; -import static pro.belbix.ethparser.web3.contracts.ContractConstants.DEPLOYERS; +import static pro.belbix.ethparser.web3.contracts.ContractConstantsV7.DEPLOYERS; import java.util.List; import lombok.extern.log4j.Log4j2; diff --git a/src/main/java/pro/belbix/ethparser/web3/deployer/transform/UniqueTypes.java b/src/main/java/pro/belbix/ethparser/web3/deployer/transform/UniqueTypes.java index f0e31e49..bdecfa57 100644 --- a/src/main/java/pro/belbix/ethparser/web3/deployer/transform/UniqueTypes.java +++ b/src/main/java/pro/belbix/ethparser/web3/deployer/transform/UniqueTypes.java @@ -2,7 +2,7 @@ import static pro.belbix.ethparser.web3.contracts.ContractConstants.PS_ADDRESS; import static pro.belbix.ethparser.web3.contracts.ContractConstants.PS_V0_ADDRESS; -import static pro.belbix.ethparser.web3.contracts.ContractConstants.iPS_ADDRESS; +import static pro.belbix.ethparser.web3.contracts.ContractConstantsV7.iPS_ADDRESS; import java.util.Map; import pro.belbix.ethparser.web3.contracts.ContractType; diff --git a/src/main/java/pro/belbix/ethparser/web3/harvest/db/VaultActionsDBService.java b/src/main/java/pro/belbix/ethparser/web3/harvest/db/VaultActionsDBService.java index 2ed12e36..c97604b1 100644 --- a/src/main/java/pro/belbix/ethparser/web3/harvest/db/VaultActionsDBService.java +++ b/src/main/java/pro/belbix/ethparser/web3/harvest/db/VaultActionsDBService.java @@ -5,7 +5,6 @@ import static pro.belbix.ethparser.service.AbiProviderService.BSC_NETWORK; import static pro.belbix.ethparser.service.AbiProviderService.ETH_NETWORK; import static pro.belbix.ethparser.service.AbiProviderService.MATIC_NETWORK; -import static pro.belbix.ethparser.web3.contracts.ContractConstants.iPS_ADDRESS; import com.fasterxml.jackson.databind.ObjectMapper; import java.math.BigInteger; @@ -23,6 +22,7 @@ import pro.belbix.ethparser.properties.AppProperties; import pro.belbix.ethparser.repositories.v0.HarvestRepository; import pro.belbix.ethparser.repositories.v0.HarvestTvlRepository; +import pro.belbix.ethparser.web3.contracts.ContractConstantsV7; import pro.belbix.ethparser.web3.contracts.ContractUtils; import pro.belbix.ethparser.web3.contracts.db.ContractDbService; import pro.belbix.ethparser.web3.prices.PriceProvider; @@ -91,7 +91,7 @@ public void fillOwnersCount(HarvestDTO dto) { contractDbService.getAllVaults(dto.getNetwork()).stream() .map(v -> v.getContract().getAddress().toLowerCase()) .filter(v -> !ContractUtils.isPsAddress(v)) - .filter(v -> !v.equalsIgnoreCase(iPS_ADDRESS)) + .filter(v -> !v.equalsIgnoreCase(ContractConstantsV7.iPS_ADDRESS)) .collect(Collectors.toList()), dto.getBlockDate(), dto.getNetwork() diff --git a/src/main/java/pro/belbix/ethparser/web3/harvest/parser/VaultActionsParser.java b/src/main/java/pro/belbix/ethparser/web3/harvest/parser/VaultActionsParser.java index 3c965bb0..a1c5bbb1 100644 --- a/src/main/java/pro/belbix/ethparser/web3/harvest/parser/VaultActionsParser.java +++ b/src/main/java/pro/belbix/ethparser/web3/harvest/parser/VaultActionsParser.java @@ -5,7 +5,6 @@ import static pro.belbix.ethparser.web3.abi.FunctionsNames.TOTAL_SUPPLY; import static pro.belbix.ethparser.web3.abi.FunctionsNames.UNDERLYING; import static pro.belbix.ethparser.web3.contracts.ContractConstants.ZERO_ADDRESS; -import static pro.belbix.ethparser.web3.contracts.ContractConstants.iPS_ADDRESS; import static pro.belbix.ethparser.web3.contracts.ContractType.POOL; import static pro.belbix.ethparser.web3.contracts.ContractType.VAULT; @@ -27,7 +26,6 @@ import pro.belbix.ethparser.model.tx.HarvestTx; import pro.belbix.ethparser.properties.AppProperties; import pro.belbix.ethparser.properties.NetworkProperties; -import pro.belbix.ethparser.repositories.ErrorsRepository; import pro.belbix.ethparser.web3.EthBlockService; import pro.belbix.ethparser.web3.ParserInfo; import pro.belbix.ethparser.web3.Web3Functions; @@ -35,6 +33,7 @@ import pro.belbix.ethparser.web3.Web3Subscriber; import pro.belbix.ethparser.web3.abi.FunctionService; import pro.belbix.ethparser.web3.abi.FunctionsUtils; +import pro.belbix.ethparser.web3.contracts.ContractConstantsV7; import pro.belbix.ethparser.web3.contracts.ContractType; import pro.belbix.ethparser.web3.contracts.ContractUtils; import pro.belbix.ethparser.web3.contracts.db.ContractDbService; @@ -202,7 +201,7 @@ private boolean parsePs(HarvestTx harvestTx, String network) { TransactionReceipt receipt = web3Functions .fetchTransactionReceipt(harvestTx.getHash(), network); String vault = receipt.getTo(); - if (vault.equalsIgnoreCase(iPS_ADDRESS)) { + if (vault.equalsIgnoreCase(ContractConstantsV7.iPS_ADDRESS)) { return false; //not count deposit from iPS } harvestTx.setMethodName("Deposit"); @@ -378,7 +377,7 @@ private void fillUsdValues(HarvestDTO dto, String vaultHash, String network) { dto.setLastTvl(vault); dto.setLastUsdTvl((double) Math.round(vault * priceUnderlying)); dto.setUsdAmount((long) (priceUnderlying * dto.getAmount() * dto.getSharePrice())); - if (iPS_ADDRESS.equalsIgnoreCase(dto.getVaultAddress())) { + if (ContractConstantsV7.iPS_ADDRESS.equalsIgnoreCase(dto.getVaultAddress())) { dto.setTotalAmount(farmTotalAmount(dto.getBlock(), network)); } } diff --git a/src/test/java/pro/belbix/ethparser/web3/deployer/DeployerTransactionsParserTest.java b/src/test/java/pro/belbix/ethparser/web3/deployer/DeployerTransactionsParserTest.java index 80d5e988..982e4133 100644 --- a/src/test/java/pro/belbix/ethparser/web3/deployer/DeployerTransactionsParserTest.java +++ b/src/test/java/pro/belbix/ethparser/web3/deployer/DeployerTransactionsParserTest.java @@ -1,12 +1,9 @@ package pro.belbix.ethparser.web3.deployer; -import static org.junit.jupiter.api.Assertions.assertAll; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; import static pro.belbix.ethparser.TestUtils.assertModel; import static pro.belbix.ethparser.service.AbiProviderService.ETH_NETWORK; -import static pro.belbix.ethparser.web3.contracts.ContractConstants.DEPLOYERS; -import static pro.belbix.ethparser.web3.contracts.ContractConstants.FARM_TOKEN; +import static pro.belbix.ethparser.web3.contracts.ContractConstantsV7.DEPLOYERS; +import static pro.belbix.ethparser.web3.contracts.ContractConstantsV7.FARM_TOKEN; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; diff --git a/src/test/java/pro/belbix/ethparser/web3/erc20/parser/TransferParserTest.java b/src/test/java/pro/belbix/ethparser/web3/erc20/parser/TransferParserTest.java index d9240543..81128d51 100644 --- a/src/test/java/pro/belbix/ethparser/web3/erc20/parser/TransferParserTest.java +++ b/src/test/java/pro/belbix/ethparser/web3/erc20/parser/TransferParserTest.java @@ -19,7 +19,7 @@ import pro.belbix.ethparser.Application; import pro.belbix.ethparser.dto.v0.TransferDTO; import pro.belbix.ethparser.web3.Web3Functions; -import pro.belbix.ethparser.web3.contracts.ContractConstants; +import pro.belbix.ethparser.web3.contracts.ContractConstantsV7; @SpringBootTest(classes = Application.class) @ContextConfiguration @@ -33,7 +33,7 @@ public class TransferParserTest { // it is a self destructed contract @Test public void testParseFARM_OneInch() { - TransferDTO dto = parserTest(ContractConstants.FARM_TOKEN, + TransferDTO dto = parserTest(ContractConstantsV7.FARM_TOKEN, 11631545, 0, "FARM", @@ -48,7 +48,7 @@ public void testParseFARM_OneInch() { @Test public void testParseFARM_firstMint() { - TransferDTO dto = parserTest(ContractConstants.FARM_TOKEN, + TransferDTO dto = parserTest(ContractConstantsV7.FARM_TOKEN, 10776698, 0, "FARM", @@ -63,7 +63,7 @@ public void testParseFARM_firstMint() { @Test public void testParseFARM_LP_REM2() { - TransferDTO dto = parserTest(ContractConstants.FARM_TOKEN, + TransferDTO dto = parserTest(ContractConstantsV7.FARM_TOKEN, 11558046, 0, "FARM", @@ -78,7 +78,7 @@ public void testParseFARM_LP_REM2() { @Test public void testParseFARM_LP_REM() { - TransferDTO dto = parserTest(ContractConstants.FARM_TOKEN, + TransferDTO dto = parserTest(ContractConstantsV7.FARM_TOKEN, 10931588, 0, "FARM", @@ -93,7 +93,7 @@ public void testParseFARM_LP_REM() { @Test public void testParseFARM_LP_ADD() { - TransferDTO dto = parserTest(ContractConstants.FARM_TOKEN, + TransferDTO dto = parserTest(ContractConstantsV7.FARM_TOKEN, 10997139, 0, "FARM", @@ -108,7 +108,7 @@ public void testParseFARM_LP_ADD() { @Test public void testParseFARM_LP_BUY() { - TransferDTO dto = parserTest(ContractConstants.FARM_TOKEN, + TransferDTO dto = parserTest(ContractConstantsV7.FARM_TOKEN, 11366155, 0, "FARM", @@ -123,7 +123,7 @@ public void testParseFARM_LP_BUY() { @Test public void testParseFARM_LP_SELL() { - TransferDTO dto = parserTest(ContractConstants.FARM_TOKEN, + TransferDTO dto = parserTest(ContractConstantsV7.FARM_TOKEN, 11373041, 0, "FARM", @@ -138,7 +138,7 @@ public void testParseFARM_LP_SELL() { @Test public void testParseFARM_exit1() { - TransferDTO dto = parserTest(ContractConstants.FARM_TOKEN, + TransferDTO dto = parserTest(ContractConstantsV7.FARM_TOKEN, 11337723, 0, "FARM", @@ -153,7 +153,7 @@ public void testParseFARM_exit1() { @Test public void testParseFARM_exit2() { - TransferDTO dto = parserTest(ContractConstants.FARM_TOKEN, + TransferDTO dto = parserTest(ContractConstantsV7.FARM_TOKEN, 11337723, 1, "FARM", @@ -168,7 +168,7 @@ public void testParseFARM_exit2() { @Test public void testParseFARM_exit3() { - TransferDTO dto = parserTest(ContractConstants.FARM_TOKEN, + TransferDTO dto = parserTest(ContractConstantsV7.FARM_TOKEN, 11337723, 2, "FARM", @@ -183,7 +183,7 @@ public void testParseFARM_exit3() { @Test public void testParseFARM_exit4() { - TransferDTO dto = parserTest(ContractConstants.FARM_TOKEN, + TransferDTO dto = parserTest(ContractConstantsV7.FARM_TOKEN, 11337723, 5, "FARM", @@ -198,7 +198,7 @@ public void testParseFARM_exit4() { @Test public void testParseFARM_stake1() { - TransferDTO dto = parserTest(ContractConstants.FARM_TOKEN, + TransferDTO dto = parserTest(ContractConstantsV7.FARM_TOKEN, 11337691, 0, "FARM", @@ -213,7 +213,7 @@ public void testParseFARM_stake1() { @Test public void testParseFARM_stake2() { - TransferDTO dto = parserTest(ContractConstants.FARM_TOKEN, + TransferDTO dto = parserTest(ContractConstantsV7.FARM_TOKEN, 11337691, 1, "FARM", @@ -228,7 +228,7 @@ public void testParseFARM_stake2() { @Test public void testParseFARM_stake3() { - TransferDTO dto = parserTest(ContractConstants.FARM_TOKEN, + TransferDTO dto = parserTest(ContractConstantsV7.FARM_TOKEN, 11337691, 6, "FARM", @@ -243,7 +243,7 @@ public void testParseFARM_stake3() { @Test public void testParseFARM_balancer() { - TransferDTO dto = parserTest(ContractConstants.FARM_TOKEN, + TransferDTO dto = parserTest(ContractConstantsV7.FARM_TOKEN, 10777054, 1, "FARM", @@ -258,7 +258,7 @@ public void testParseFARM_balancer() { @Test public void testParseFARM_bot() { - TransferDTO dto = parserTest(ContractConstants.FARM_TOKEN, + TransferDTO dto = parserTest(ContractConstantsV7.FARM_TOKEN, 10850197, 1, "FARM", @@ -273,7 +273,7 @@ public void testParseFARM_bot() { @Test public void testParseFARM_swap() { - TransferDTO dto = parserTest(ContractConstants.FARM_TOKEN, + TransferDTO dto = parserTest(ContractConstantsV7.FARM_TOKEN, 11055960, 0, "FARM", @@ -288,7 +288,7 @@ public void testParseFARM_swap() { @Test public void testParseFARM_addLiquidity() { - TransferDTO dto = parserTest(ContractConstants.FARM_TOKEN, + TransferDTO dto = parserTest(ContractConstantsV7.FARM_TOKEN, 11362801, 1, "FARM", @@ -303,7 +303,7 @@ public void testParseFARM_addLiquidity() { @Test public void testParseFARM_transfer() { - parserTest(ContractConstants.FARM_TOKEN, + parserTest(ContractConstantsV7.FARM_TOKEN, 11571359, 0, "FARM", @@ -317,7 +317,7 @@ public void testParseFARM_transfer() { @Test public void testParseFARM_HARD_WORK() { - parserTest(ContractConstants.FARM_TOKEN, + parserTest(ContractConstantsV7.FARM_TOKEN, 11045532, 0, "FARM", @@ -331,7 +331,7 @@ public void testParseFARM_HARD_WORK() { @Test public void testParseFARM_HARD_WORK2() { - parserTest(ContractConstants.FARM_TOKEN, + parserTest(ContractConstantsV7.FARM_TOKEN, 11045532, 1, "FARM", @@ -345,7 +345,7 @@ public void testParseFARM_HARD_WORK2() { @Test public void testParseFARM_swapExactTokensForETH() { - parserTest(ContractConstants.FARM_TOKEN, + parserTest(ContractConstantsV7.FARM_TOKEN, 10777107, 0, "FARM", diff --git a/src/test/java/pro/belbix/ethparser/web3/harvest/parser/ImportantEventsParserTest.java b/src/test/java/pro/belbix/ethparser/web3/harvest/parser/ImportantEventsParserTest.java index d0a7b7bc..b5a2e12d 100644 --- a/src/test/java/pro/belbix/ethparser/web3/harvest/parser/ImportantEventsParserTest.java +++ b/src/test/java/pro/belbix/ethparser/web3/harvest/parser/ImportantEventsParserTest.java @@ -19,7 +19,7 @@ import pro.belbix.ethparser.Application; import pro.belbix.ethparser.dto.v0.ImportantEventsDTO; import pro.belbix.ethparser.web3.Web3Functions; -import pro.belbix.ethparser.web3.contracts.ContractConstants; +import pro.belbix.ethparser.web3.contracts.ContractConstantsV7; @SpringBootTest(classes = Application.class) @ContextConfiguration @@ -59,7 +59,7 @@ public void shouldParseStrategyAnnounce() { @Test public void shouldParseMint() { parserTest( - ContractConstants.FARM_TOKEN, + ContractConstantsV7.FARM_TOKEN, 10776715, 0, "null", diff --git a/src/test/java/pro/belbix/ethparser/web3/harvest/vault/VaultActionsParserEthTest.java b/src/test/java/pro/belbix/ethparser/web3/harvest/vault/VaultActionsParserEthTest.java index 9e33ff4a..6ce722cc 100644 --- a/src/test/java/pro/belbix/ethparser/web3/harvest/vault/VaultActionsParserEthTest.java +++ b/src/test/java/pro/belbix/ethparser/web3/harvest/vault/VaultActionsParserEthTest.java @@ -10,7 +10,7 @@ import static pro.belbix.ethparser.service.AbiProviderService.ETH_NETWORK; import static pro.belbix.ethparser.web3.contracts.ContractConstants.PS_ADDRESS; import static pro.belbix.ethparser.web3.contracts.ContractConstants.PS_V0_ADDRESS; -import static pro.belbix.ethparser.web3.contracts.ContractConstants.iPS_ADDRESS; +import static pro.belbix.ethparser.web3.contracts.ContractConstantsV7.iPS_ADDRESS; import java.util.List; import org.junit.jupiter.api.Disabled; diff --git a/src/test/java/pro/belbix/ethparser/web3/layers/detector/EthContractDetectorTest.java b/src/test/java/pro/belbix/ethparser/web3/layers/detector/EthContractDetectorTest.java index d68f7683..2ef55147 100644 --- a/src/test/java/pro/belbix/ethparser/web3/layers/detector/EthContractDetectorTest.java +++ b/src/test/java/pro/belbix/ethparser/web3/layers/detector/EthContractDetectorTest.java @@ -7,7 +7,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static pro.belbix.ethparser.TestUtils.assertTwoArrays; import static pro.belbix.ethparser.service.AbiProviderService.ETH_NETWORK; -import static pro.belbix.ethparser.web3.contracts.ContractConstants.FARM_TOKEN; +import static pro.belbix.ethparser.web3.contracts.ContractConstantsV7.FARM_TOKEN; import com.fasterxml.jackson.core.JsonProcessingException; import java.util.ArrayList; From 37a125aa1321177fc99df1307e00bf31e817e381 Mon Sep 17 00:00:00 2001 From: alex Date: Wed, 27 Apr 2022 00:08:12 +0300 Subject: [PATCH 58/65] Try to resolve problem with NoClassDefFoundError --- .../web3/contracts/ContractConstants.java | 40 ------------------- .../web3/contracts/ContractConstantsV7.java | 17 ++++++++ .../web3/contracts/ContractUtils.java | 2 +- .../web3/harvest/parser/RewardParser.java | 3 +- 4 files changed, 19 insertions(+), 43 deletions(-) diff --git a/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstants.java b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstants.java index f18e52bc..2ba4203f 100644 --- a/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstants.java +++ b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstants.java @@ -68,46 +68,6 @@ public class ContractConstants { .toLowerCase(); public static final String BANCOR_CONVERSION_ADDRESS = "0x2f9ec37d6ccfff1cab21733bdadede11c823ccb0"; - final static Map> CONTROLLERS = Map.of( - ETH_NETWORK, Map.of( - 10770087L, "0x222412af183BCeAdEFd72e4Cb1b71f1889953b1C".toLowerCase(), - 12652885L, "0x3cc47874dc50d98425ec79e647d83495637c55e3".toLowerCase()), - BSC_NETWORK, Map.of( - 5990839L, "0x222412af183bceadefd72e4cb1b71f1889953b1c".toLowerCase()), - MATIC_NETWORK, Map.of( - 16612698L, "0x2ce34b1bb247f242f1d2a33811e01138968efbff".toLowerCase() - ) - ); - - public final static Map NOTIFY_HELPER = Map.of( - ETH_NETWORK, "0xe20c31e3d08027f5aface84a3a46b7b3b165053c".toLowerCase(), - BSC_NETWORK, "0xf71042c88458ff1702c3870f62f4c764712cc9f0".toLowerCase(), - MATIC_NETWORK, "0xe85c8581e60d7cd32bbfd86303d2a4fa6a951dac".toLowerCase() - ); - - final static Map> ORACLES = Map.of( - ETH_NETWORK, - Map.of(12015724L, "0x48DC32eCA58106f06b41dE514F29780FFA59c279".toLowerCase(), - 12820106L, "0x1358c91D5b25D3eDAc2b7B26A619163d78f1717d".toLowerCase()), - BSC_NETWORK, - Map.of(6442627L, "0xE0e9F05054Ad3a2b6414AD13D768be91a84b47e8".toLowerCase(), - 6952687L, "0x643cF46eef91Bd878D9710ceEB6a7E6F929F2608".toLowerCase(), - 9142012L, "0x0E74303d0D18884Ce2CEb3670e72686645c4f38B".toLowerCase()), - MATIC_NETWORK, - Map.of(16841617L, "0x0E74303d0D18884Ce2CEb3670e72686645c4f38B".toLowerCase()) - ); - - final static Map> ORACLES_BY_FACTORY = Map.of( - ETH_NETWORK, - Map.of(UNISWAP_FACTORY_ADDRESS, - "0x48DC32eCA58106f06b41dE514F29780FFA59c279".toLowerCase()), - BSC_NETWORK, - Map.of(PCS_V1_FACTORY_ADDRESS, - "0xE0e9F05054Ad3a2b6414AD13D768be91a84b47e8".toLowerCase(), // V1 - PCS_V2_FACTORY_ADDRESS, - "0x643cF46eef91Bd878D9710ceEB6a7E6F929F2608".toLowerCase()) // V2 - ); - static final Map> FULL_PARSABLE_UNI_PAIRS = Map.of( ETH_NETWORK, Map.of( "0x514906fc121c7878424a5c928cad1852cc545892".toLowerCase(), 10777067, diff --git a/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstantsV7.java b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstantsV7.java index 30304eae..c1bc05d5 100644 --- a/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstantsV7.java +++ b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstantsV7.java @@ -57,4 +57,21 @@ public interface ContractConstantsV7 { BSC_NETWORK, "0xf00dd244228f51547f0563e60bca65a30fbf5f7f".toLowerCase(), MATIC_NETWORK, "0xf00dd244228f51547f0563e60bca65a30fbf5f7f".toLowerCase() ); + + Map> CONTROLLERS = Map.of( + ETH_NETWORK, Map.of( + 10770087L, "0x222412af183BCeAdEFd72e4Cb1b71f1889953b1C".toLowerCase(), + 12652885L, "0x3cc47874dc50d98425ec79e647d83495637c55e3".toLowerCase()), + BSC_NETWORK, Map.of( + 5990839L, "0x222412af183bceadefd72e4cb1b71f1889953b1c".toLowerCase()), + MATIC_NETWORK, Map.of( + 16612698L, "0x2ce34b1bb247f242f1d2a33811e01138968efbff".toLowerCase() + ) + ); + + Map NOTIFY_HELPER = Map.of( + ETH_NETWORK, "0xe20c31e3d08027f5aface84a3a46b7b3b165053c".toLowerCase(), + BSC_NETWORK, "0xf71042c88458ff1702c3870f62f4c764712cc9f0".toLowerCase(), + MATIC_NETWORK, "0xe85c8581e60d7cd32bbfd86303d2a4fa6a951dac".toLowerCase() + ); } diff --git a/src/main/java/pro/belbix/ethparser/web3/contracts/ContractUtils.java b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractUtils.java index 520a6dd7..1b5390bf 100644 --- a/src/main/java/pro/belbix/ethparser/web3/contracts/ContractUtils.java +++ b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractUtils.java @@ -5,7 +5,7 @@ import static pro.belbix.ethparser.service.AbiProviderService.MATIC_NETWORK; import static pro.belbix.ethparser.web3.contracts.ContractConstants.BSC_BLOCK_NUMBER_18_MARCH_2021; import static pro.belbix.ethparser.web3.contracts.ContractConstants.BSC_FARM_TOKEN; -import static pro.belbix.ethparser.web3.contracts.ContractConstants.CONTROLLERS; +import static pro.belbix.ethparser.web3.contracts.ContractConstantsV7.CONTROLLERS; import static pro.belbix.ethparser.web3.contracts.ContractConstants.ETH_BLOCK_NUMBER_30_AUGUST_2020; import static pro.belbix.ethparser.web3.contracts.ContractConstantsV7.FARM_TOKEN; import static pro.belbix.ethparser.web3.contracts.ContractConstants.FULL_PARSABLE_UNI_PAIRS; diff --git a/src/main/java/pro/belbix/ethparser/web3/harvest/parser/RewardParser.java b/src/main/java/pro/belbix/ethparser/web3/harvest/parser/RewardParser.java index 0ae6c720..2119bdb0 100644 --- a/src/main/java/pro/belbix/ethparser/web3/harvest/parser/RewardParser.java +++ b/src/main/java/pro/belbix/ethparser/web3/harvest/parser/RewardParser.java @@ -5,10 +5,10 @@ import static pro.belbix.ethparser.web3.abi.FunctionsNames.PERIOD_FINISH; import static pro.belbix.ethparser.web3.abi.FunctionsNames.REWARD_RATE; import static pro.belbix.ethparser.web3.contracts.ContractConstants.D18; -import static pro.belbix.ethparser.web3.contracts.ContractConstants.NOTIFY_HELPER; import static pro.belbix.ethparser.web3.contracts.ContractConstants.PS_ADDRESS; import static pro.belbix.ethparser.web3.contracts.ContractConstants.PS_V0_ADDRESS; import static pro.belbix.ethparser.web3.contracts.ContractConstants.ST_PS_ADDRESS; +import static pro.belbix.ethparser.web3.contracts.ContractConstantsV7.NOTIFY_HELPER; import static pro.belbix.ethparser.web3.contracts.ContractType.POOL; import java.math.BigDecimal; @@ -23,7 +23,6 @@ import pro.belbix.ethparser.model.tx.HarvestTx; import pro.belbix.ethparser.properties.AppProperties; import pro.belbix.ethparser.properties.NetworkProperties; -import pro.belbix.ethparser.repositories.ErrorsRepository; import pro.belbix.ethparser.web3.EthBlockService; import pro.belbix.ethparser.web3.ParserInfo; import pro.belbix.ethparser.web3.Web3Functions; From 44badf9d2b4378c6505019b20597fa0006d5ce03 Mon Sep 17 00:00:00 2001 From: alex Date: Wed, 27 Apr 2022 00:18:08 +0300 Subject: [PATCH 59/65] Try to resolve problem with NoClassDefFoundError --- .../web3/bancor/BancorPriceParser.java | 2 +- .../web3/contracts/ContractConstants.java | 118 ------------------ .../web3/contracts/ContractConstantsV7.java | 9 ++ .../web3/contracts/ContractConstantsV8.java | 81 ++++++++++++ .../web3/contracts/ContractLoader.java | 4 +- .../web3/contracts/ContractUtils.java | 4 +- .../ethparser/web3/prices/LPSeeker.java | 8 +- .../ethparser/web3/prices/PriceOracle.java | 4 +- .../web3/bancor/BancorPriceParserTest.java | 2 +- 9 files changed, 102 insertions(+), 130 deletions(-) create mode 100644 src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstantsV8.java diff --git a/src/main/java/pro/belbix/ethparser/web3/bancor/BancorPriceParser.java b/src/main/java/pro/belbix/ethparser/web3/bancor/BancorPriceParser.java index 2c52ebaa..2cab4620 100644 --- a/src/main/java/pro/belbix/ethparser/web3/bancor/BancorPriceParser.java +++ b/src/main/java/pro/belbix/ethparser/web3/bancor/BancorPriceParser.java @@ -3,7 +3,7 @@ import static pro.belbix.ethparser.service.AbiProviderService.ETH_NETWORK; import static pro.belbix.ethparser.web3.abi.FunctionsNames.RATE_BY_PATH; import static pro.belbix.ethparser.web3.contracts.ContractConstants.BANCOR_CONVERSION_ADDRESS; -import static pro.belbix.ethparser.web3.contracts.ContractConstants.BANCOR_USDC_CONVERT_PATH; +import static pro.belbix.ethparser.web3.contracts.ContractConstantsV8.BANCOR_USDC_CONVERT_PATH; import static pro.belbix.ethparser.web3.contracts.ContractConstants.D6; import static pro.belbix.ethparser.web3.contracts.ContractConstantsV7.FARM_TOKEN; import static pro.belbix.ethparser.web3.contracts.ContractConstants.L18; diff --git a/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstants.java b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstants.java index 2ba4203f..950de2b0 100644 --- a/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstants.java +++ b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstants.java @@ -1,13 +1,6 @@ package pro.belbix.ethparser.web3.contracts; -import static pro.belbix.ethparser.service.AbiProviderService.BSC_NETWORK; -import static pro.belbix.ethparser.service.AbiProviderService.ETH_NETWORK; -import static pro.belbix.ethparser.service.AbiProviderService.MATIC_NETWORK; - import java.math.BigInteger; -import java.util.List; -import java.util.Map; -import java.util.Set; import org.web3j.protocol.core.DefaultBlockParameter; import org.web3j.protocol.core.DefaultBlockParameterNumber; @@ -31,14 +24,6 @@ public class ContractConstants { public static final DefaultBlockParameterNumber MATIC_BLOCK_NUMBER_06_JUL_2021 = (DefaultBlockParameterNumber) DefaultBlockParameter.valueOf(new BigInteger("16566542")); - public static final String PCS_V1_FACTORY_ADDRESS = "0xbcfccbde45ce874adcb698cc183debcf17952812" - .toLowerCase(); - public static final String PCS_V2_FACTORY_ADDRESS = "0xca143ce32fe78f1f7019d7d551a6402fc5350c73" - .toLowerCase(); - public static final String UNISWAP_FACTORY_ADDRESS = "0x5C69bEe701ef814a2B6a3EDD4B1652CB9cc5aA6f" - .toLowerCase(); - public static final String SUSHISWAP_FACTORY_ADDRESS = "0xC0AEe478e3658e2610c5F7A4A2E1777cE9e4f2Ac" - .toLowerCase(); public static final String CURVE_REGISTRY_ADDRESS = "0x7D86446dDb609eD0F5f8684AcF30380a356b2B4c" .toLowerCase(); public static final String BELT_POOL_ADDRESS = "0xF16D312d119c13dD27fD0dC814b0bCdcaAa62dfD" @@ -62,108 +47,5 @@ public class ContractConstants { public static final String ST_PS_ADDRESS = "0x8f5adC58b32D4e5Ca02EAC0E293D35855999436C" .toLowerCase(); - public static final String QUICK_FACTORY_ADDRESS = "0x5757371414417b8C6CAad45bAeF941aBc7d3Ab32" - .toLowerCase(); - public static final String SUSHI_FACTORY_ADDRESS = "0xc35dadb65012ec5796536bd9864ed8773abc74c4" - .toLowerCase(); public static final String BANCOR_CONVERSION_ADDRESS = "0x2f9ec37d6ccfff1cab21733bdadede11c823ccb0"; - - static final Map> FULL_PARSABLE_UNI_PAIRS = Map.of( - ETH_NETWORK, Map.of( - "0x514906fc121c7878424a5c928cad1852cc545892".toLowerCase(), 10777067, - // UNI_LP_USDC_FARM - FARM - "0x56feaccb7f750b997b36a68625c7c596f0b41a58".toLowerCase(), 11407437, - // UNI_LP_WETH_FARM - FARM - "0xb9fa44b0911f6d777faab2fa9d8ef103f25ddf49".toLowerCase(), 11407202 - // UNI_LP_GRAIN_FARM - GRAIN - ), - BSC_NETWORK, Map.of(), - MATIC_NETWORK, Map.of() - ); - - public static final List BANCOR_USDC_CONVERT_PATH = List.of( - "0x1F573D6Fb3F13d689FF844B4cE37794d79a7FF1C".toLowerCase(), // BNT - "0x874d8dE5b26c9D9f6aA8d7bab283F9A9c6f777f4".toLowerCase(), // Liquidity Pool (USDCBNT) - "0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48".toLowerCase() // USDC - ); - - static final Map> PARSABLE_BANCOR_TRANSACTIONS = Map.of( - ETH_NETWORK, Map.of( - BANCOR_CONVERSION_ADDRESS, 10285676 - ), - BSC_NETWORK, Map.of(), - MATIC_NETWORK, Map.of() - ); - - // TODO separate by networks - public static final Set ONE_DOLLAR_TOKENS = Set.of( - "0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48".toLowerCase(), //USDC - "0xe9e7cea3dedca5984780bafc599bd69add087d56".toLowerCase(), //BUSD - "0xdAC17F958D2ee523a2206206994597C13D831ec7".toLowerCase(), //USDT - "0x0000000000085d4780B73119b644AE5ecd22b376".toLowerCase(), //TUSD - "0x2791bca1f2de4661ed88a30c99a7a9449aa84174".toLowerCase(), //matic USDC - "0xc2132d05d31c914a87c6611c10748aeb04b58e8f".toLowerCase(), //matic USDT - "0xE840B73E5287865EEc17d250bFb1536704B43B21".toLowerCase(), //matic mUSD - "0x2791Bca1f2de4661ED88A30C99A7a9449Aa84174".toLowerCase() //matic USDC PoS - ); - - //Key tokens are used to find liquidity for any given token on Uni, Sushi and Curve. - public static final Map> KEY_TOKENS = Map.of( - ETH_NETWORK, Set.of( - "0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48".toLowerCase(), //USDC - "0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2".toLowerCase(), //WETH - "0x6B175474E89094C44Da98b954EedeAC495271d0F".toLowerCase(), //DAI - "0xdAC17F958D2ee523a2206206994597C13D831ec7".toLowerCase(), //USDT - "0xa47c8bf37f92aBed4A126BDA807A7b7498661acD".toLowerCase(), //UST - "0x2260FAC5E5542a773Aa44fBCfeDf7C193bc2C599".toLowerCase(), //WBTC - "0xdB25f211AB05b1c97D595516F45794528a807ad8".toLowerCase(), //EURS - "0x514910771AF9Ca656af840dff83E8264EcF986CA".toLowerCase() //LINK - ), - BSC_NETWORK, Set.of( - "0x8AC76a51cc950d9822D68b83fE1Ad97B32Cd580d".toLowerCase(), //USDC - "0x2170Ed0880ac9A755fd29B2688956BD959F933F8".toLowerCase(), //ETH - "0x1AF3F329e8BE154074D8769D1FFa4eE058B1DBc3".toLowerCase(), //DAI - "0x55d398326f99059fF775485246999027B3197955".toLowerCase(), //USDT - "0x23396cF899Ca06c4472205fC903bDB4de249D6fC".toLowerCase(), //UST - "0x7130d2A12B9BCbFAe4f2634d864A1Ee1Ce3Ead9c".toLowerCase(), //BTCB - "0xe9e7CEA3DedcA5984780Bafc599bD69ADd087D56".toLowerCase(), //BUSD - "0xbb4CdB9CBd36B01bD1cBaEBF2De08d9173bc095c".toLowerCase(), //WBNB - "0x4BD17003473389A42DAF6a0a729f6Fdb328BbBd7".toLowerCase(), //VAI - "0x111111111117dC0aa78b770fA6A738034120C302".toLowerCase() //1INCH - ), - MATIC_NETWORK, Set.of( - "0x2791bca1f2de4661ed88a30c99a7a9449aa84174".toLowerCase(), //USDC - "0x7ceb23fd6bc0add59e62ac25578270cff1b9f619".toLowerCase(), //ETH - "0x8f3cf7ad23cd3cadbd9735aff958023239c6a063".toLowerCase(), //DAI - "0xc2132d05d31c914a87c6611c10748aeb04b58e8f".toLowerCase(), //USDT - "0x692597b009d13c4049a947cab2239b7d6517875f".toLowerCase(), //UST - "0xdab529f40e671a1d4bf91361c21bf9f0c9712ab7".toLowerCase(), //BUSD - "0x1bfd67037b42cf73acf2047067bd4f2c47d9bfd6".toLowerCase() //WBTC - ) - ); - - //Pricing tokens are Key tokens with good liquidity with the defined output token on Uniswap. - public static final Map> PRISING_TOKENS = Map.of( - ETH_NETWORK, Set.of( - "0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48".toLowerCase(), //USDC - "0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2".toLowerCase(), //WETH - "0x6B175474E89094C44Da98b954EedeAC495271d0F".toLowerCase(), //DAI - "0xdAC17F958D2ee523a2206206994597C13D831ec7".toLowerCase(), //USDT - "0x2260FAC5E5542a773Aa44fBCfeDf7C193bc2C599".toLowerCase(), //WBTC - "0xdB25f211AB05b1c97D595516F45794528a807ad8".toLowerCase() //EURS - ) - ); - - public static final Map> UNI_FACTORIES = Map.of( - ETH_NETWORK, Map.of( - UNISWAP_FACTORY_ADDRESS, 10000835, - SUSHISWAP_FACTORY_ADDRESS, 10794229 - ), BSC_NETWORK, Map.of( - PCS_V1_FACTORY_ADDRESS, 586851, - PCS_V2_FACTORY_ADDRESS, 6809737 - ), MATIC_NETWORK, Map.of( - QUICK_FACTORY_ADDRESS, 4931780, - SUSHI_FACTORY_ADDRESS, 11333218 - ) - ); } diff --git a/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstantsV7.java b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstantsV7.java index c1bc05d5..b71dc7ec 100644 --- a/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstantsV7.java +++ b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstantsV7.java @@ -12,6 +12,7 @@ public interface ContractConstantsV7 { .toLowerCase(); String FARM_TOKEN = "0xa0246c9032bc3a600820415ae600c6388619a14d" .toLowerCase(); + String BANCOR_CONVERSION_ADDRESS = "0x2f9ec37d6ccfff1cab21733bdadede11c823ccb0"; Map COIN_PRICE_IN_OTHER_CHAIN = Map.of( // Denarius (DEN-0121) to Denarius BSC @@ -74,4 +75,12 @@ public interface ContractConstantsV7 { BSC_NETWORK, "0xf71042c88458ff1702c3870f62f4c764712cc9f0".toLowerCase(), MATIC_NETWORK, "0xe85c8581e60d7cd32bbfd86303d2a4fa6a951dac".toLowerCase() ); + + Map> PARSABLE_BANCOR_TRANSACTIONS = Map.of( + ETH_NETWORK, Map.of( + BANCOR_CONVERSION_ADDRESS, 10285676 + ), + BSC_NETWORK, Map.of(), + MATIC_NETWORK, Map.of() + ); } diff --git a/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstantsV8.java b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstantsV8.java new file mode 100644 index 00000000..3b3163ac --- /dev/null +++ b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractConstantsV8.java @@ -0,0 +1,81 @@ +package pro.belbix.ethparser.web3.contracts; + +import static pro.belbix.ethparser.service.AbiProviderService.BSC_NETWORK; +import static pro.belbix.ethparser.service.AbiProviderService.ETH_NETWORK; +import static pro.belbix.ethparser.service.AbiProviderService.MATIC_NETWORK; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +public interface ContractConstantsV8 { + + String QUICK_FACTORY_ADDRESS = "0x5757371414417b8C6CAad45bAeF941aBc7d3Ab32" + .toLowerCase(); + String SUSHI_FACTORY_ADDRESS = "0xc35dadb65012ec5796536bd9864ed8773abc74c4" + .toLowerCase(); + String PCS_V1_FACTORY_ADDRESS = "0xbcfccbde45ce874adcb698cc183debcf17952812" + .toLowerCase(); + String PCS_V2_FACTORY_ADDRESS = "0xca143ce32fe78f1f7019d7d551a6402fc5350c73" + .toLowerCase(); + String UNISWAP_FACTORY_ADDRESS = "0x5C69bEe701ef814a2B6a3EDD4B1652CB9cc5aA6f" + .toLowerCase(); + String SUSHISWAP_FACTORY_ADDRESS = "0xC0AEe478e3658e2610c5F7A4A2E1777cE9e4f2Ac" + .toLowerCase(); + + + List BANCOR_USDC_CONVERT_PATH = List.of( + "0x1F573D6Fb3F13d689FF844B4cE37794d79a7FF1C".toLowerCase(), // BNT + "0x874d8dE5b26c9D9f6aA8d7bab283F9A9c6f777f4".toLowerCase(), // Liquidity Pool (USDCBNT) + "0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48".toLowerCase() // USDC + ); + + + //Key tokens are used to find liquidity for any given token on Uni, Sushi and Curve. + Map> KEY_TOKENS = Map.of( + ETH_NETWORK, Set.of( + "0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48".toLowerCase(), //USDC + "0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2".toLowerCase(), //WETH + "0x6B175474E89094C44Da98b954EedeAC495271d0F".toLowerCase(), //DAI + "0xdAC17F958D2ee523a2206206994597C13D831ec7".toLowerCase(), //USDT + "0xa47c8bf37f92aBed4A126BDA807A7b7498661acD".toLowerCase(), //UST + "0x2260FAC5E5542a773Aa44fBCfeDf7C193bc2C599".toLowerCase(), //WBTC + "0xdB25f211AB05b1c97D595516F45794528a807ad8".toLowerCase(), //EURS + "0x514910771AF9Ca656af840dff83E8264EcF986CA".toLowerCase() //LINK + ), + BSC_NETWORK, Set.of( + "0x8AC76a51cc950d9822D68b83fE1Ad97B32Cd580d".toLowerCase(), //USDC + "0x2170Ed0880ac9A755fd29B2688956BD959F933F8".toLowerCase(), //ETH + "0x1AF3F329e8BE154074D8769D1FFa4eE058B1DBc3".toLowerCase(), //DAI + "0x55d398326f99059fF775485246999027B3197955".toLowerCase(), //USDT + "0x23396cF899Ca06c4472205fC903bDB4de249D6fC".toLowerCase(), //UST + "0x7130d2A12B9BCbFAe4f2634d864A1Ee1Ce3Ead9c".toLowerCase(), //BTCB + "0xe9e7CEA3DedcA5984780Bafc599bD69ADd087D56".toLowerCase(), //BUSD + "0xbb4CdB9CBd36B01bD1cBaEBF2De08d9173bc095c".toLowerCase(), //WBNB + "0x4BD17003473389A42DAF6a0a729f6Fdb328BbBd7".toLowerCase(), //VAI + "0x111111111117dC0aa78b770fA6A738034120C302".toLowerCase() //1INCH + ), + MATIC_NETWORK, Set.of( + "0x2791bca1f2de4661ed88a30c99a7a9449aa84174".toLowerCase(), //USDC + "0x7ceb23fd6bc0add59e62ac25578270cff1b9f619".toLowerCase(), //ETH + "0x8f3cf7ad23cd3cadbd9735aff958023239c6a063".toLowerCase(), //DAI + "0xc2132d05d31c914a87c6611c10748aeb04b58e8f".toLowerCase(), //USDT + "0x692597b009d13c4049a947cab2239b7d6517875f".toLowerCase(), //UST + "0xdab529f40e671a1d4bf91361c21bf9f0c9712ab7".toLowerCase(), //BUSD + "0x1bfd67037b42cf73acf2047067bd4f2c47d9bfd6".toLowerCase() //WBTC + ) + ); + + Map> UNI_FACTORIES = Map.of( + ETH_NETWORK, Map.of( + UNISWAP_FACTORY_ADDRESS, 10000835, + SUSHISWAP_FACTORY_ADDRESS, 10794229 + ), BSC_NETWORK, Map.of( + PCS_V1_FACTORY_ADDRESS, 586851, + PCS_V2_FACTORY_ADDRESS, 6809737 + ), MATIC_NETWORK, Map.of( + QUICK_FACTORY_ADDRESS, 4931780, + SUSHI_FACTORY_ADDRESS, 11333218 + ) + ); +} diff --git a/src/main/java/pro/belbix/ethparser/web3/contracts/ContractLoader.java b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractLoader.java index 5bbe75dd..96ebc17e 100644 --- a/src/main/java/pro/belbix/ethparser/web3/contracts/ContractLoader.java +++ b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractLoader.java @@ -15,8 +15,8 @@ import static pro.belbix.ethparser.web3.contracts.ContractConstants.PAIR_TYPE_ONEINCHE; import static pro.belbix.ethparser.web3.contracts.ContractConstants.PAIR_TYPE_SUSHI; import static pro.belbix.ethparser.web3.contracts.ContractConstants.PAIR_TYPE_UNISWAP; -import static pro.belbix.ethparser.web3.contracts.ContractConstants.SUSHISWAP_FACTORY_ADDRESS; -import static pro.belbix.ethparser.web3.contracts.ContractConstants.UNISWAP_FACTORY_ADDRESS; +import static pro.belbix.ethparser.web3.contracts.ContractConstantsV8.SUSHISWAP_FACTORY_ADDRESS; +import static pro.belbix.ethparser.web3.contracts.ContractConstantsV8.UNISWAP_FACTORY_ADDRESS; import static pro.belbix.ethparser.web3.contracts.ContractConstants.ZERO_ADDRESS; import static pro.belbix.ethparser.web3.contracts.ContractType.INFRASTRUCTURE; diff --git a/src/main/java/pro/belbix/ethparser/web3/contracts/ContractUtils.java b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractUtils.java index 1b5390bf..d130dcde 100644 --- a/src/main/java/pro/belbix/ethparser/web3/contracts/ContractUtils.java +++ b/src/main/java/pro/belbix/ethparser/web3/contracts/ContractUtils.java @@ -8,12 +8,12 @@ import static pro.belbix.ethparser.web3.contracts.ContractConstantsV7.CONTROLLERS; import static pro.belbix.ethparser.web3.contracts.ContractConstants.ETH_BLOCK_NUMBER_30_AUGUST_2020; import static pro.belbix.ethparser.web3.contracts.ContractConstantsV7.FARM_TOKEN; -import static pro.belbix.ethparser.web3.contracts.ContractConstants.FULL_PARSABLE_UNI_PAIRS; +import static pro.belbix.ethparser.web3.contracts.ContractConstantsV2.FULL_PARSABLE_UNI_PAIRS; import static pro.belbix.ethparser.web3.contracts.ContractConstants.MATIC_BLOCK_NUMBER_06_JUL_2021; import static pro.belbix.ethparser.web3.contracts.ContractConstants.MATIC_FARM_TOKEN; import static pro.belbix.ethparser.web3.contracts.ContractConstants.ONE_INCH_FACTORY_ADDRESS; import static pro.belbix.ethparser.web3.contracts.ContractConstants.ONE_INCH_FACTORY_BSC; -import static pro.belbix.ethparser.web3.contracts.ContractConstants.PARSABLE_BANCOR_TRANSACTIONS; +import static pro.belbix.ethparser.web3.contracts.ContractConstantsV7.PARSABLE_BANCOR_TRANSACTIONS; import static pro.belbix.ethparser.web3.contracts.ContractConstants.PS_V0_ADDRESS; import static pro.belbix.ethparser.web3.contracts.ContractConstants.ZERO_ADDRESS; diff --git a/src/main/java/pro/belbix/ethparser/web3/prices/LPSeeker.java b/src/main/java/pro/belbix/ethparser/web3/prices/LPSeeker.java index 7f6475b6..b11bab2d 100644 --- a/src/main/java/pro/belbix/ethparser/web3/prices/LPSeeker.java +++ b/src/main/java/pro/belbix/ethparser/web3/prices/LPSeeker.java @@ -4,7 +4,7 @@ import static pro.belbix.ethparser.web3.abi.FunctionsNames.GET_PAIR; import static pro.belbix.ethparser.web3.abi.FunctionsNames.GET_RESERVES; import static pro.belbix.ethparser.web3.abi.FunctionsNames.TOKEN0; -import static pro.belbix.ethparser.web3.contracts.ContractConstants.UNI_FACTORIES; +import static pro.belbix.ethparser.web3.contracts.ContractConstantsV8.UNI_FACTORIES; import static pro.belbix.ethparser.web3.contracts.ContractConstants.ZERO_ADDRESS; import java.math.BigDecimal; @@ -20,7 +20,7 @@ import org.web3j.abi.datatypes.Function; import org.web3j.protocol.ObjectMapperFactory; import pro.belbix.ethparser.web3.abi.FunctionsUtils; -import pro.belbix.ethparser.web3.contracts.ContractConstants; +import pro.belbix.ethparser.web3.contracts.ContractConstantsV8; import pro.belbix.ethparser.web3.contracts.ContractUtils; import pro.belbix.ethparser.web3.contracts.db.ContractDbService; import pro.belbix.ethparser.web3.contracts.models.PureEthContractInfo; @@ -70,7 +70,7 @@ private String getUniLargestPool( String network, List contracts ) { - Set tokenList = ContractConstants.KEY_TOKENS.get(network); + Set tokenList = ContractConstantsV8.KEY_TOKENS.get(network); TreeMap pairsLiquidity = new TreeMap<>(); for (String keyToken : tokenList) { for (String factory : UNI_FACTORIES.get(network).keySet()) { @@ -103,7 +103,7 @@ private boolean isEligibleKeyToken(String tokenAddress, String keyToken, String return false; } boolean tokenIsKeyToken = - ContractConstants.KEY_TOKENS.get(network).contains(tokenAddress.toLowerCase()); + ContractConstantsV8.KEY_TOKENS.get(network).contains(tokenAddress.toLowerCase()); boolean keyTokenIsStablecoin = ContractUtils.isStableCoin(keyToken); // for avoid recursion we should have keyToken -> Stablecoin pairs only return !tokenIsKeyToken || keyTokenIsStablecoin; diff --git a/src/main/java/pro/belbix/ethparser/web3/prices/PriceOracle.java b/src/main/java/pro/belbix/ethparser/web3/prices/PriceOracle.java index 2f6a130e..df9f1aba 100644 --- a/src/main/java/pro/belbix/ethparser/web3/prices/PriceOracle.java +++ b/src/main/java/pro/belbix/ethparser/web3/prices/PriceOracle.java @@ -17,7 +17,7 @@ import org.web3j.protocol.ObjectMapperFactory; import pro.belbix.ethparser.properties.AppProperties; import pro.belbix.ethparser.web3.abi.FunctionsUtils; -import pro.belbix.ethparser.web3.contracts.ContractConstants; +import pro.belbix.ethparser.web3.contracts.ContractConstantsV8; import pro.belbix.ethparser.web3.contracts.ContractUtils; @Service @@ -64,7 +64,7 @@ public double getPriceForCoinOnChain(String tokenAdr, Long block, String network public String getLargestKeyToken(String tokenAddress, long block, String network) { String oracleAddress = getOracleAddress(tokenAddress, block, network); - List
tokenList = ContractConstants.KEY_TOKENS.get(network).stream() + List
tokenList = ContractConstantsV8.KEY_TOKENS.get(network).stream() .map(Address::new) .collect(Collectors.toList()); try { diff --git a/src/test/java/pro/belbix/ethparser/web3/bancor/BancorPriceParserTest.java b/src/test/java/pro/belbix/ethparser/web3/bancor/BancorPriceParserTest.java index 0ca3415a..23a4fc7f 100644 --- a/src/test/java/pro/belbix/ethparser/web3/bancor/BancorPriceParserTest.java +++ b/src/test/java/pro/belbix/ethparser/web3/bancor/BancorPriceParserTest.java @@ -7,7 +7,7 @@ import static pro.belbix.ethparser.service.AbiProviderService.ETH_NETWORK; import static pro.belbix.ethparser.web3.abi.FunctionsNames.RATE_BY_PATH; import static pro.belbix.ethparser.web3.contracts.ContractConstants.BANCOR_CONVERSION_ADDRESS; -import static pro.belbix.ethparser.web3.contracts.ContractConstants.BANCOR_USDC_CONVERT_PATH; +import static pro.belbix.ethparser.web3.contracts.ContractConstantsV8.BANCOR_USDC_CONVERT_PATH; import static pro.belbix.ethparser.web3.contracts.ContractConstants.D6; import static pro.belbix.ethparser.web3.contracts.ContractConstants.L18; import static pro.belbix.ethparser.web3.contracts.ContractUtils.isParsableBancorTransaction; From 803b6bd014979c27f8a738fdb0df61bf787eb69a Mon Sep 17 00:00:00 2001 From: alex Date: Wed, 27 Apr 2022 20:21:22 +0300 Subject: [PATCH 60/65] Fix problem with one more result from db query --- .../ethparser/web3/contracts/db/ContractDbService.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/pro/belbix/ethparser/web3/contracts/db/ContractDbService.java b/src/main/java/pro/belbix/ethparser/web3/contracts/db/ContractDbService.java index d22fcc72..ad49a01e 100644 --- a/src/main/java/pro/belbix/ethparser/web3/contracts/db/ContractDbService.java +++ b/src/main/java/pro/belbix/ethparser/web3/contracts/db/ContractDbService.java @@ -57,8 +57,10 @@ public ContractDbService( } public Optional getContractByAddress(String address, String network) { - return Optional.ofNullable(contractRepository - .findFirstByAddress(address.toLowerCase(), network)); + return contractRepository + .findFirstByAddress(address.toLowerCase(), network, PageRequest.of(0, 1)) + .stream() + .findFirst(); } public Optional getContractByAddressAndType( From 82759c788403064a813890cfaec534a1330cd970 Mon Sep 17 00:00:00 2001 From: alex Date: Thu, 28 Apr 2022 09:46:21 +0300 Subject: [PATCH 61/65] Add more logs for HardWorkParser --- .../ethparser/web3/harvest/parser/HardWorkParser.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/main/java/pro/belbix/ethparser/web3/harvest/parser/HardWorkParser.java b/src/main/java/pro/belbix/ethparser/web3/harvest/parser/HardWorkParser.java index b7450af3..828ffa87 100644 --- a/src/main/java/pro/belbix/ethparser/web3/harvest/parser/HardWorkParser.java +++ b/src/main/java/pro/belbix/ethparser/web3/harvest/parser/HardWorkParser.java @@ -205,6 +205,8 @@ public HardWorkDTO parse(Log ethLog, String network) { // // skip old strategies // return null; // } + + log.info("OldSharePrice: {}, NewSharePrice: {}", tx.getOldSharePrice(), tx.getNewSharePrice()); HardWorkDTO dto = new HardWorkDTO(); dto.setNetwork(network); dto.setId(tx.getHash() + "_" + tx.getLogId()); @@ -356,7 +358,7 @@ private void parseRewardAddedEventsEth( return; } double reward = tx.getReward().doubleValue() / D18; - + log.info("parseRewardAddedEventsEth: {} {} reward is {}", dto.getVaultAddress(), dto.getNetwork(), reward); // AutoStake strategies have two RewardAdded events - first for PS and second for stake contract if (autoStake && dto.getFarmBuyback() != 0) { // in this case it is second reward for strategy @@ -364,6 +366,7 @@ private void parseRewardAddedEventsEth( / (1 - requireNonNullElse(dto.getProfitSharingRate(), defaultPsDenominator(network))); // full reward dto.setFullRewardUsd(fullReward); + log.info("fullReward: {}", fullReward); } else { double farmBuybackMultiplier = (1 - requireNonNullElse(dto.getProfitSharingRate(), defaultPsDenominator(network))) @@ -373,6 +376,7 @@ private void parseRewardAddedEventsEth( // PS pool reward dto.setFarmBuyback(reward + (reward * farmBuybackMultiplier)); + log.info("It's autoStake - {}", autoStake); // for non AutoStake strategy we will not have accurate data for strategy reward // just calculate aprox value based on PS reward if (!autoStake) { @@ -380,6 +384,7 @@ private void parseRewardAddedEventsEth( / requireNonNullElse(dto.getProfitSharingRate(), defaultPsDenominator(network))); // full reward dto.setFullRewardUsd(fullReward); + log.info("fullReward: {}", fullReward); } } From a2e23b393fb7a9bf3428b95dcee432a0ebb8601c Mon Sep 17 00:00:00 2001 From: alex Date: Thu, 28 Apr 2022 10:00:09 +0300 Subject: [PATCH 62/65] Add logs for check NullPointerException --- .../ethparser/web3/Web3TransactionFlowable.java | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/main/java/pro/belbix/ethparser/web3/Web3TransactionFlowable.java b/src/main/java/pro/belbix/ethparser/web3/Web3TransactionFlowable.java index b8f2330e..f05e9403 100644 --- a/src/main/java/pro/belbix/ethparser/web3/Web3TransactionFlowable.java +++ b/src/main/java/pro/belbix/ethparser/web3/Web3TransactionFlowable.java @@ -73,14 +73,15 @@ public void run() { AtomicInteger counter = new AtomicInteger(0); web3Functions.findBlocksByBlockBatch(from, to, network) .sorted(Comparator.comparing(Block::getNumber)) - .forEach(block -> - transactionConsumers.forEach(queue -> - block.getTransactions().forEach(t -> { - counter.incrementAndGet(); - writeInQueue(queue, (Transaction) t.get(), transactionConsumers.size()); - }) - ) - ); + .forEach(block -> { + log.info("Block info: {}, transactionConsumers: {}", block, transactionConsumers); + transactionConsumers.forEach(queue -> + block.getTransactions().forEach(t -> { + counter.incrementAndGet(); + writeInQueue(queue, (Transaction) t.get(), transactionConsumers.size()); + }) + ); + }); lastParsedBlock = to; saveLast(to); log.info("Parse {} transactions from {} to {} on block: {} - {}", From 7c18d1050eda42287cdbb818e734dc42a8f51002 Mon Sep 17 00:00:00 2001 From: alex Date: Thu, 28 Apr 2022 16:51:45 +0300 Subject: [PATCH 63/65] Add additional info from harvest api --- scripts/application.example.yml | 3 + scripts/sql/schema.sql | 14 +++++ .../controllers/HardWorkController.java | 30 ++++++++- .../ethparser/dto/v0/HardWorkHarvestDTO.java | 53 ++++++++++++++++ .../ethparser/entity/HarvestVaultData.java | 31 ++++++++++ .../ethparser/model/HarvestVaultInfo.java | 5 ++ .../HarvestVaultDataRepository.java | 8 +++ .../service/HarvestVaultDataService.java | 61 +++++++++++++++++++ .../service/external/HarvestService.java | 1 - .../service/task/HarvestUpdateInfoTask.java | 45 ++++++++++++++ src/main/resources/application.yml | 5 +- 11 files changed, 252 insertions(+), 4 deletions(-) create mode 100644 src/main/java/pro/belbix/ethparser/dto/v0/HardWorkHarvestDTO.java create mode 100644 src/main/java/pro/belbix/ethparser/entity/HarvestVaultData.java create mode 100644 src/main/java/pro/belbix/ethparser/repositories/HarvestVaultDataRepository.java create mode 100644 src/main/java/pro/belbix/ethparser/service/HarvestVaultDataService.java create mode 100644 src/main/java/pro/belbix/ethparser/service/task/HarvestUpdateInfoTask.java diff --git a/scripts/application.example.yml b/scripts/application.example.yml index 3e8f0021..c29dcde1 100644 --- a/scripts/application.example.yml +++ b/scripts/application.example.yml @@ -42,3 +42,6 @@ task: max-thread-size: 30 fixedRate: 86400000 # Everyday enable: false + info: + fixedRate: 3600000 # Each hour + enable: false \ No newline at end of file diff --git a/scripts/sql/schema.sql b/scripts/sql/schema.sql index f8b2c29c..84d9259a 100644 --- a/scripts/sql/schema.sql +++ b/scripts/sql/schema.sql @@ -1135,3 +1135,17 @@ create table token_price value numeric(60, 6) ); +create table harvest_vault_data +( + id varchar(255) not null + constraint harvest_vault_data_pk + primary key, + vault_address varchar(255) not null, + reward_pool varchar(255), + display_name varchar(255), + network varchar(99), + apy double precision not null, + tvl double precision not null, + total_supply double precision not null +); + diff --git a/src/main/java/pro/belbix/ethparser/controllers/HardWorkController.java b/src/main/java/pro/belbix/ethparser/controllers/HardWorkController.java index 4d43265f..26e5415b 100644 --- a/src/main/java/pro/belbix/ethparser/controllers/HardWorkController.java +++ b/src/main/java/pro/belbix/ethparser/controllers/HardWorkController.java @@ -13,6 +13,7 @@ import io.swagger.v3.oas.annotations.responses.ApiResponses; import io.swagger.v3.oas.annotations.tags.Tag; import java.util.List; +import java.util.stream.Collectors; import lombok.extern.log4j.Log4j2; import org.apache.logging.log4j.util.Strings; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; @@ -27,10 +28,13 @@ import org.springframework.web.bind.annotation.RestController; import org.web3j.protocol.ObjectMapperFactory; import pro.belbix.ethparser.dto.v0.HardWorkDTO; +import pro.belbix.ethparser.dto.v0.HardWorkHarvestDTO; +import pro.belbix.ethparser.entity.HarvestVaultData; import pro.belbix.ethparser.model.PaginatedResponse; import pro.belbix.ethparser.model.RestResponse; import pro.belbix.ethparser.repositories.v0.HardWorkRepository; import pro.belbix.ethparser.service.DtoCache; +import pro.belbix.ethparser.service.HarvestVaultDataService; import pro.belbix.ethparser.web3.contracts.ContractType; import pro.belbix.ethparser.web3.contracts.db.ContractDbService; import pro.belbix.ethparser.web3.harvest.HardWorkCalculator; @@ -45,14 +49,18 @@ public class HardWorkController { private final HardWorkCalculator hardWorkCalculator; private final ContractDbService contractDbService; private final DtoCache dtoCache; + private final HarvestVaultDataService harvestVaultDataService; public HardWorkController(HardWorkRepository hardWorkRepository, HardWorkCalculator hardWorkCalculator, - ContractDbService contractDbService, DtoCache dtoCache) { + ContractDbService contractDbService, DtoCache dtoCache, + HarvestVaultDataService harvestVaultDataService + ) { this.hardWorkRepository = hardWorkRepository; this.hardWorkCalculator = hardWorkCalculator; this.contractDbService = contractDbService; this.dtoCache = dtoCache; + this.harvestVaultDataService = harvestVaultDataService; } @Operation(summary = "Returns the latest data for HardWorks") @@ -272,6 +280,11 @@ public RestResponse hardworkPages( if (!pages.hasContent()) { return RestResponse.error("Data not found"); } + var vaultInfo = harvestVaultDataService.getAllVaultInfo(); + var content = pages.getContent().stream() + .map(i -> toHardWorkHarvestDTO(i, vaultInfo)) + .collect(Collectors.toList()); + return RestResponse.ok( ObjectMapperFactory.getObjectMapper().writeValueAsString( PaginatedResponse.builder() @@ -279,7 +292,7 @@ public RestResponse hardworkPages( .previousPage(pages.hasPrevious() ? start - 1 : -1) .nextPage(pages.hasNext() ? start + 1 : -1) .totalPages(pages.getTotalPages()) - .data(pages.getContent()) + .data(content) .build() ) ); @@ -292,4 +305,17 @@ public RestResponse hardworkPages( } + private HardWorkHarvestDTO toHardWorkHarvestDTO(HardWorkDTO dto, List items) { + var value = new HardWorkHarvestDTO(dto); + var item = items.stream() + .filter(i -> i.getVaultAddress().equalsIgnoreCase(dto.getVaultAddress()) && i.getNetwork().equalsIgnoreCase(dto.getNetwork())) + .findFirst(); + if (item.isPresent()) { + value.setApy(item.get().getApy()); + value.setTvl(item.get().getTvl()); + } + + return value; + } + } diff --git a/src/main/java/pro/belbix/ethparser/dto/v0/HardWorkHarvestDTO.java b/src/main/java/pro/belbix/ethparser/dto/v0/HardWorkHarvestDTO.java new file mode 100644 index 00000000..8d43f8e7 --- /dev/null +++ b/src/main/java/pro/belbix/ethparser/dto/v0/HardWorkHarvestDTO.java @@ -0,0 +1,53 @@ +package pro.belbix.ethparser.dto.v0; + +import java.math.BigDecimal; +import lombok.Data; +import lombok.EqualsAndHashCode; + +@EqualsAndHashCode(callSuper = true) +@Data +public class HardWorkHarvestDTO extends HardWorkDTO { + BigDecimal apy; + BigDecimal tvl; + + public HardWorkHarvestDTO(HardWorkDTO hardWorkDTO) { + + id = hardWorkDTO.id; + vault = hardWorkDTO.vault; + vaultAddress = hardWorkDTO.vaultAddress; + block = hardWorkDTO.block; + blockDate = hardWorkDTO.blockDate; + network = hardWorkDTO.network; + shareChange = hardWorkDTO.shareChange; + fullRewardUsd = hardWorkDTO.fullRewardUsd; + farmBuyback = hardWorkDTO.farmBuyback; + fee = hardWorkDTO.fee; + farmBuybackEth = hardWorkDTO.farmBuybackEth; + gasUsed = hardWorkDTO.gasUsed; + invested = hardWorkDTO.invested; + investmentTarget = hardWorkDTO.investmentTarget; + farmPrice = hardWorkDTO.farmPrice; + ethPrice = hardWorkDTO.ethPrice; + profitSharingRate = hardWorkDTO.profitSharingRate; + buyBackRate = hardWorkDTO.buyBackRate; + autoStake = hardWorkDTO.autoStake; + + idleTime = hardWorkDTO.idleTime; + feeEth = hardWorkDTO.feeEth; + savedGasFeesSum = hardWorkDTO.savedGasFeesSum; + savedGasFees = hardWorkDTO.savedGasFees; + poolUsers = hardWorkDTO.poolUsers; + callsQuantity = hardWorkDTO.callsQuantity; + farmBuybackSum = hardWorkDTO.farmBuybackSum; + psApr = hardWorkDTO.psApr; + psTvlUsd = hardWorkDTO.psTvlUsd; + weeklyProfit = hardWorkDTO.weeklyProfit; + + weeklyAllProfit = hardWorkDTO.weeklyAllProfit; + apr = hardWorkDTO.apr; + perc = hardWorkDTO.perc; + fullRewardUsdTotal = hardWorkDTO.fullRewardUsdTotal; + allProfit = hardWorkDTO.allProfit; + + } +} diff --git a/src/main/java/pro/belbix/ethparser/entity/HarvestVaultData.java b/src/main/java/pro/belbix/ethparser/entity/HarvestVaultData.java new file mode 100644 index 00000000..4aa891bb --- /dev/null +++ b/src/main/java/pro/belbix/ethparser/entity/HarvestVaultData.java @@ -0,0 +1,31 @@ +package pro.belbix.ethparser.entity; + +import java.math.BigDecimal; +import javax.persistence.Entity; +import javax.persistence.Id; +import javax.persistence.Table; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.FieldDefaults; + +@Entity +@Table(name = "harvest_vault_data") +@Data +@FieldDefaults(level = AccessLevel.PRIVATE) +@Builder +@AllArgsConstructor +@NoArgsConstructor +public class HarvestVaultData { + @Id + String id; + String vaultAddress; + String rewardPool; + String displayName; + BigDecimal apy; + BigDecimal tvl; + BigDecimal totalSupply; + String network; +} diff --git a/src/main/java/pro/belbix/ethparser/model/HarvestVaultInfo.java b/src/main/java/pro/belbix/ethparser/model/HarvestVaultInfo.java index 58227b80..7b0d9d88 100644 --- a/src/main/java/pro/belbix/ethparser/model/HarvestVaultInfo.java +++ b/src/main/java/pro/belbix/ethparser/model/HarvestVaultInfo.java @@ -1,6 +1,7 @@ package pro.belbix.ethparser.model; import com.fasterxml.jackson.annotation.JsonProperty; +import java.math.BigDecimal; import java.util.Map; import lombok.Data; @@ -16,9 +17,13 @@ public class HarvestVaultInfo { @Data public static class HarvestVaultItemInfo { + private int chain; private String vaultAddress; private String id; private String rewardPool; private String displayName; + private BigDecimal estimatedApy; + private BigDecimal totalValueLocked; + private BigDecimal totalSupply; } } diff --git a/src/main/java/pro/belbix/ethparser/repositories/HarvestVaultDataRepository.java b/src/main/java/pro/belbix/ethparser/repositories/HarvestVaultDataRepository.java new file mode 100644 index 00000000..3de2f258 --- /dev/null +++ b/src/main/java/pro/belbix/ethparser/repositories/HarvestVaultDataRepository.java @@ -0,0 +1,8 @@ +package pro.belbix.ethparser.repositories; + +import org.springframework.data.jpa.repository.JpaRepository; +import pro.belbix.ethparser.entity.HarvestVaultData; + +public interface HarvestVaultDataRepository extends JpaRepository { + +} diff --git a/src/main/java/pro/belbix/ethparser/service/HarvestVaultDataService.java b/src/main/java/pro/belbix/ethparser/service/HarvestVaultDataService.java new file mode 100644 index 00000000..6168017d --- /dev/null +++ b/src/main/java/pro/belbix/ethparser/service/HarvestVaultDataService.java @@ -0,0 +1,61 @@ +package pro.belbix.ethparser.service; + + +import static pro.belbix.ethparser.service.AbiProviderService.BSC_NETWORK; +import static pro.belbix.ethparser.service.AbiProviderService.ETH_NETWORK; +import static pro.belbix.ethparser.service.AbiProviderService.MATIC_NETWORK; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; +import lombok.experimental.FieldDefaults; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import pro.belbix.ethparser.entity.HarvestVaultData; +import pro.belbix.ethparser.model.HarvestVaultInfo.HarvestVaultItemInfo; +import pro.belbix.ethparser.repositories.HarvestVaultDataRepository; + +@Service +@RequiredArgsConstructor +@Slf4j +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) +public class HarvestVaultDataService { + private final static Map CHAIN_BY_NETWORK = Map.of( + 137, MATIC_NETWORK, + 1, ETH_NETWORK, + 56, BSC_NETWORK + ); + + HarvestVaultDataRepository harvestVaultDataRepository; + + public List saveAll(List items) { + return harvestVaultDataRepository.saveAll( + items.stream() + .map(this::toHarvestVaultData) + .collect(Collectors.toList()) + ); + } + + public HarvestVaultData save(HarvestVaultItemInfo harvestVaultInfo) { + return harvestVaultDataRepository.save(toHarvestVaultData(harvestVaultInfo)); + } + + public List getAllVaultInfo() { + return harvestVaultDataRepository.findAll(); + } + + private HarvestVaultData toHarvestVaultData(HarvestVaultItemInfo harvestVaultInfo) { + return HarvestVaultData.builder() + .id(harvestVaultInfo.getId()) + .vaultAddress(harvestVaultInfo.getVaultAddress()) + .displayName(harvestVaultInfo.getDisplayName()) + .rewardPool(harvestVaultInfo.getRewardPool()) + .apy(harvestVaultInfo.getEstimatedApy()) + .totalSupply(harvestVaultInfo.getTotalSupply()) + .tvl(harvestVaultInfo.getTotalValueLocked()) + .network(CHAIN_BY_NETWORK.get(harvestVaultInfo.getChain())) + .build(); + } +} diff --git a/src/main/java/pro/belbix/ethparser/service/external/HarvestService.java b/src/main/java/pro/belbix/ethparser/service/external/HarvestService.java index a31f2b30..9976546c 100644 --- a/src/main/java/pro/belbix/ethparser/service/external/HarvestService.java +++ b/src/main/java/pro/belbix/ethparser/service/external/HarvestService.java @@ -29,7 +29,6 @@ public HarvestVaultInfo getVaults() { var url = String.format(HarvestUrl.VAULTS, externalProperties.getHarvest().getUrl(), externalProperties.getHarvest().getKey()); log.info("Starting get vaults from harvest {} ", url); var result = restTemplate.getForEntity(url, HarvestVaultInfo.class); - log.info("Result getting vaults from harvest API : {}", result); return result.getBody(); } } diff --git a/src/main/java/pro/belbix/ethparser/service/task/HarvestUpdateInfoTask.java b/src/main/java/pro/belbix/ethparser/service/task/HarvestUpdateInfoTask.java new file mode 100644 index 00000000..536bffd5 --- /dev/null +++ b/src/main/java/pro/belbix/ethparser/service/task/HarvestUpdateInfoTask.java @@ -0,0 +1,45 @@ +package pro.belbix.ethparser.service.task; + +import java.util.Collection; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import pro.belbix.ethparser.service.HarvestVaultDataService; +import pro.belbix.ethparser.service.external.HarvestService; + +@Service +@RequiredArgsConstructor +@Slf4j +public class HarvestUpdateInfoTask { + @Value("${task.info.enable}") + private Boolean enable; + private final HarvestService harvestService; + private final HarvestVaultDataService harvestVaultDataService; + + @Scheduled(fixedRateString = "${task.info.fixedRate}") + public void start() { + if (enable == null || !enable) { + log.info("HarvestUpdateInfoTask disabled"); + return; + } + + log.info("Start save vaults info from harvest API"); + var response = harvestService.getVaults(); + + var vaults = Stream.of( + response.getEthereumNetwork().values(), + response.getBscNetwork().values(), + response.getMaticNetwork().values() + ) + .flatMap(Collection::stream) + .collect(Collectors.toList()); + + + harvestVaultDataService.saveAll(vaults); + log.info("Success saved vaults info from harvest API"); + } +} diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 512ad06e..1e4300da 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -74,7 +74,7 @@ task: enable: false vault: fixedRate: 3600000 # Each hour - enable: true + enable: false uni-pair: fixedRate: 86400000 # Everyday enable: false @@ -82,5 +82,8 @@ task: max-thread-size: 30 fixedRate: 86400000 # Everyday enable: false + info: + fixedRate: 3600000 # Each hour + enable: false From 4a22b03e199c2ad02f1db6e1c8891ff11284eab8 Mon Sep 17 00:00:00 2001 From: alex Date: Thu, 28 Apr 2022 22:58:41 +0300 Subject: [PATCH 64/65] Add new endpoint with harvest vault info --- .../HarvestVaultDataController.java | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 src/main/java/pro/belbix/ethparser/controllers/HarvestVaultDataController.java diff --git a/src/main/java/pro/belbix/ethparser/controllers/HarvestVaultDataController.java b/src/main/java/pro/belbix/ethparser/controllers/HarvestVaultDataController.java new file mode 100644 index 00000000..58b85dea --- /dev/null +++ b/src/main/java/pro/belbix/ethparser/controllers/HarvestVaultDataController.java @@ -0,0 +1,26 @@ +package pro.belbix.ethparser.controllers; + +import java.util.List; +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; +import lombok.experimental.FieldDefaults; +import org.springframework.http.HttpStatus; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.bind.annotation.RestController; +import pro.belbix.ethparser.entity.HarvestVaultData; +import pro.belbix.ethparser.service.HarvestVaultDataService; + +@RestController +@RequiredArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true) +public class HarvestVaultDataController { + HarvestVaultDataService harvestVaultDataService; + + @GetMapping("/harvest/vaults") + @ResponseStatus(HttpStatus.OK) + public List getAll() { + return harvestVaultDataService.getAllVaultInfo(); + } + +} From 016aa4bce749fe491c9b03c237ef92c5f2d2e97d Mon Sep 17 00:00:00 2001 From: alex Date: Fri, 29 Apr 2022 00:21:59 +0300 Subject: [PATCH 65/65] Remove unused feature --- .../controllers/HardWorkController.java | 30 ++----------------- 1 file changed, 2 insertions(+), 28 deletions(-) diff --git a/src/main/java/pro/belbix/ethparser/controllers/HardWorkController.java b/src/main/java/pro/belbix/ethparser/controllers/HardWorkController.java index 26e5415b..f08510c0 100644 --- a/src/main/java/pro/belbix/ethparser/controllers/HardWorkController.java +++ b/src/main/java/pro/belbix/ethparser/controllers/HardWorkController.java @@ -13,7 +13,6 @@ import io.swagger.v3.oas.annotations.responses.ApiResponses; import io.swagger.v3.oas.annotations.tags.Tag; import java.util.List; -import java.util.stream.Collectors; import lombok.extern.log4j.Log4j2; import org.apache.logging.log4j.util.Strings; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; @@ -28,13 +27,10 @@ import org.springframework.web.bind.annotation.RestController; import org.web3j.protocol.ObjectMapperFactory; import pro.belbix.ethparser.dto.v0.HardWorkDTO; -import pro.belbix.ethparser.dto.v0.HardWorkHarvestDTO; -import pro.belbix.ethparser.entity.HarvestVaultData; import pro.belbix.ethparser.model.PaginatedResponse; import pro.belbix.ethparser.model.RestResponse; import pro.belbix.ethparser.repositories.v0.HardWorkRepository; import pro.belbix.ethparser.service.DtoCache; -import pro.belbix.ethparser.service.HarvestVaultDataService; import pro.belbix.ethparser.web3.contracts.ContractType; import pro.belbix.ethparser.web3.contracts.db.ContractDbService; import pro.belbix.ethparser.web3.harvest.HardWorkCalculator; @@ -49,18 +45,15 @@ public class HardWorkController { private final HardWorkCalculator hardWorkCalculator; private final ContractDbService contractDbService; private final DtoCache dtoCache; - private final HarvestVaultDataService harvestVaultDataService; public HardWorkController(HardWorkRepository hardWorkRepository, HardWorkCalculator hardWorkCalculator, - ContractDbService contractDbService, DtoCache dtoCache, - HarvestVaultDataService harvestVaultDataService + ContractDbService contractDbService, DtoCache dtoCache ) { this.hardWorkRepository = hardWorkRepository; this.hardWorkCalculator = hardWorkCalculator; this.contractDbService = contractDbService; this.dtoCache = dtoCache; - this.harvestVaultDataService = harvestVaultDataService; } @Operation(summary = "Returns the latest data for HardWorks") @@ -280,11 +273,6 @@ public RestResponse hardworkPages( if (!pages.hasContent()) { return RestResponse.error("Data not found"); } - var vaultInfo = harvestVaultDataService.getAllVaultInfo(); - var content = pages.getContent().stream() - .map(i -> toHardWorkHarvestDTO(i, vaultInfo)) - .collect(Collectors.toList()); - return RestResponse.ok( ObjectMapperFactory.getObjectMapper().writeValueAsString( PaginatedResponse.builder() @@ -292,7 +280,7 @@ public RestResponse hardworkPages( .previousPage(pages.hasPrevious() ? start - 1 : -1) .nextPage(pages.hasNext() ? start + 1 : -1) .totalPages(pages.getTotalPages()) - .data(content) + .data(pages.getContent()) .build() ) ); @@ -304,18 +292,4 @@ public RestResponse hardworkPages( } } - - private HardWorkHarvestDTO toHardWorkHarvestDTO(HardWorkDTO dto, List items) { - var value = new HardWorkHarvestDTO(dto); - var item = items.stream() - .filter(i -> i.getVaultAddress().equalsIgnoreCase(dto.getVaultAddress()) && i.getNetwork().equalsIgnoreCase(dto.getNetwork())) - .findFirst(); - if (item.isPresent()) { - value.setApy(item.get().getApy()); - value.setTvl(item.get().getTvl()); - } - - return value; - } - }