diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000000..effc37cbb8 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,14 @@ +root = true + +[*] +insert_final_newline = true +charset = utf-8 +trim_trailing_whitespace = true +end_of_line = lf + +[*.{js,json,ts,tsx}] +indent_style = space +indent_size = 2 + +[*.{md,markdown}] +trim_trailing_whitespace = false diff --git a/docs/guides/reputation-mining-client.md b/docs/guides/reputation-mining-client.md deleted file mode 100644 index 619a57f92e..0000000000 --- a/docs/guides/reputation-mining-client.md +++ /dev/null @@ -1,129 +0,0 @@ ---- -description: How to use the Reputation Mining Client -sidebar_position: 3 ---- - -# Reputation Mining Client - -## Running the Mining Client - -The reputation mining client can be run locally to sync with a local ganache instance, the `goerli` testnet, or with glider on `mainnet`. - -To participate in the reputation mining process you need to have staked at least the [minimum amount of CLNY Tokens](../interfaces/ireputationminingcycle#getminstake-uint256-minstake), for at least [one full mining cycle duration](../interfaces/ireputationminingcycle#getminingwindowduration-uint256-miningwindowduration) before you can submit a new reputation root hash. - -Usage: - -```bash -node packages/reputation-miner/bin/index.js (--arguments ) [--arguments ] -``` - -Mandatory arguments: - -``` -(--minerAddress
) | (--privateKey ) -(--colonyNetworkAddress
) -(--syncFrom ) // [goerli:'548534', mainnet:'7913100'] -``` - -Optional arguments: - -``` -[--network <(goerli|mainnet)>] -[--localPort ] -[--dbPath <$PATH>] -[--auto <(true|false)>] -``` - -#### `--minerAddress` -Address of the miner account which the client will send reputation mining contract transactions from. Used when working with an unlocked account for the miner against **development networks only**. We provision twelve unlocked test accounts stored in `ganache-accounts.json` for testing that are available when starting a local ganache-cli instance via `npm run start:blockchain:client` command. - -#### `--privateKey` - -Private key of the miner account which the client will sign reputation mining contract transactions with. - -#### `--colonyNetworkAddress` - -The address of the Colony Network's `EtherRouter`. See [Upgrades to the Colony Network](../concepts/upgrades) for more information about the EtherRouter design pattern. This address is static on `goerli` and `mainnet` `goerli` `0x79073fc2117dD054FCEdaCad1E7018C9CbE3ec0B` `mainnet` `0x5346d0f80e2816fad329f2c140c870ffc3c3e2ef` - -#### `--dbPath` - -Path for the sqlite database storing reputation state. Default is `./reputationStates.sqlite`. - -#### `--network` - -Used for connecting to a supported Infura node (instead of a local client). Valid options are `goerli` and `mainnet`. - -#### `--localPort` - -Used to connect to a local clinet running on the specified port. Default is `8545`. - -#### `--syncFrom` - -Block number to start reputation state sync from. This is the block at which the reputation mining process was initialised. This number is static on `goerli` and `mainnet` - -* `goerli: 548534` -* `mainnet: 7913100` - -Note that beginning the sync with a too-early block will result in an error. If you get this exception, try syncing from a more recent block. Note that the sync process can take long. Latest tests syncing a client from scratch to 28 reputation cycles took \~2 hours. - -#### `--auto` - -Default is `true` - -The "auto" reputation mining client will: - -* Propose a new hash at the first possible block time, and submit until the maximum number has been reached (based on staked CLNY, with a maximum of 12 submissions allowed) -* Respond to challenges if there are disagreeing submissions. -* Confirm the last hash after the mining window closes and any disputes have been resolved. - -Reputation mining protocol details can be found in the [Whitepaper TLDR](../tldr/reputation-mining). - -## Visualizations - -The reputation mining client comes with a set of built-in visualizers to make it easier to view reputation states and to see the current state of the mining process. Once a mining client is running and connected to a network, navigate to the client's address in a browser (i.e. `http://localhost:3000/`) to access the available visualization tools. - -### Force Reputation Updates - -The client is set to provide a reputation update once every 24 hours. For testing, you'll likely want to "fast-forward" your network through a few submissions to see usable reputation. - -You can move the network forward by 24 hours with the following command. - -```bash -curl -H "Content-Type: application/json" -X POST --data '{"jsonrpc":"2.0","method":"evm_increaseTime","params":[86400],"id": 1}' localhost:8545 -``` - -Once you have moved the network forward 24 hours, you can then mine a new block with the following command. - -```bash -curl -H "Content-Type: application/json" -X POST --data '{"jsonrpc":"2.0","method":"evm_mine","params":[]}' localhost:8545 -``` - -Note that because reputation is awarded for the _previous_ submission window, you will need to use the "fast-forward" command above to speed through at least 2 reputation updates before noticing a change in the miner's reputation. - -## Get Reputation from the Reputation Oracle - -The reputation mining client will answer queries for reputation scores locally over HTTP. - -``` -http://127.0.0.1:3000/{reputationState}/{colonyAddress}/{skillId}/{userAddress} -``` - -An instance of the oracle is available for reputation queries against `goerli` or `mainnet` networks: - -``` -https://xdai.colony.io/reputation/{network}/{reputationState}/{colonyAddress}/{skillId}/{userAddress} -``` - -The oracle should be able to provide responses to any valid reputation score in all historical states, as well as the current state. For querying the colony-wide reputation instead of user-specific one, instead of {userAddress} use a zero address (`0x0000000000000000000000000000000000000000`) - -For example, you can get the reputation score of the miner in a reputation state `0xc7eb2cf60aa4848ce0feed5d713c07fd26e404dd50ca3b9e4f2fabef196ca3bc`) using the address of the Meta Colony (`0x14946533cefe742399e9734a123f0c02d0405a51`), the mining skill id (`2`), and address of a miner (`0x0A1d439C7d0b9244035d4F934BBF8A418B35d064`). - -``` -https://xdai.colony.io/reputation/mainnet/0xc7eb2cf60aa4848ce0feed5d713c07fd26e404dd50ca3b9e4f2fabef196ca3bc/0x14946533cefe742399e9734a123f0c02d0405a51/2/0x0A1d439C7d0b9244035d4F934BBF8A418B35d064 -``` - -The oracle returns - -``` -{"branchMask":"0xc000000000000000000000000000000000000000000000000000000000000000","siblings":["0x15c45d734bccc204df2e275d516250ed0a1cd60ccabadf49e2157a3e8067e59c","0xd4ee79473ec5573d706be030f3077c44aef06f26745349bbd93dcf5f4e254422"],"key":"0x14946533cefe742399e9734a123f0c02d0405a5100000000000000000000000000000000000000000000000000000000000000020a1d439c7d0b9244035d4f934bbf8a418b35d064","value":"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004","reputation":"0x0000000000000000000000000000000000000000000000000000000000000000","uid":"0x0000000000000000000000000000000000000000000000000000000000000004","reputationAmount":"0"} -``` diff --git a/docs/guides/reputation-mining.md b/docs/guides/reputation-mining.md index 63fea5abcb..7bb29aca6d 100644 --- a/docs/guides/reputation-mining.md +++ b/docs/guides/reputation-mining.md @@ -3,22 +3,48 @@ description: A guide on how to set up a reputation miner sidebar_position: 2 --- +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + # Reputation Mining -#### A. Introduction +## Introduction + +Colony's reputation system is key to its functionality, and in order to work successfully, with decays being calculated and new reputation being awarded as appropriate, it relies on the reputation 'mining' process. Any user with sufficient CLNY can stake that CLNY to participate in the process. This guide is intended to provide information for a (sufficiently technical) user who wishes to do so. For more information on the concept of Reputation Mining, please see [this document](../tldr/reputation-mining.md). + +## Getting started with Reputation Mining + +### A. Staking and awarding appropriate permissions on-chain + +To participate in the reputation mining process you need to have staked at least the minimum amount of CLNY Tokens (currently 2000 CLNY), for at least one full mining cycle duration (currently 60 minutes) before you can submit a new reputation root hash. The stake can be slashed if malicious behavior is detected. + +Optionally, you can define an Ethreum address to delegate the mining to. In this way the address that is mining doesn't need to be the one that is staking (as the private key needs to be stored when using the Mining Client). -Colony's reputation system is key to its functionality, and in order to work successfully, with decays being calculated and new reputation being awarded as appropriate, it relies on the reputation 'mining' process. Any user with sufficient CLNY can stake that CLNY to participate in the process. This guide is intended to provide information for a (sufficiently technical) user who wishes to do so. +:::info +Delegating the miner (using two addresses) like this is not strictly required, but represents best practices - even if your mining process ended up compromised and the private key accessed, the worst that could be done with it would be to slowly have your stake slashed by misbehaving, which can be prevented by removing the permissions of the disposable address (see below) +::: + + + + +Connect this site to MetaMask and use the following form to add stake to use for Reputation Mining. The only step that is mandatory is to stake at least 2000 CLNY which is the minimum stake required to participate in Reputation Mining. -To participate in the reputation mining process you need to have staked at least the minimum amount of CLNY Tokens (currently 2000 CLNY), for at least one full mining cycle duration (currently 60 minutes) before you can submit a new reputation root hash. +**We recommend using a separate Ethereum account for the miner itself and set it as a delegate.** This can be done in the second step of the form. -#### B. Awarding appropriate permissions on-chain +Shall you wish to stop participating in Reputation Mining, you can unstake your CLNY in the third step. -1\. Check out our contract repository, following [these instructions](../docs/quick-start.md#cloning-the-repository-and-preparing-the-dependencies). You should then be able to run `yarn run truffle console --network xdai` which will connect you to the right network. You will need to be able to sign messages from the address in control of your CLNY (which will also be the address earning reputation for mining), which in most cases means pasting your private key into `truffle.js` before launching the console. For Ledger support, you can use `yarn run truffle console --network xdaiLedger`. For other hardware wallets, you will need to find an appropriate provider compatible with Truffle, and add it into `truffle.js` in your local version of the repository.\ -\ -An appropriate gas price for the current level of network use can be found at [https://blockscout.com/xdai/mainnet/](https://blockscout.com/xdai/mainnet/). The default value in `truffle.js` represent 2Gwei.\ + + -2\. Create references to the various contracts that we will need. Run each of these commands in turn: + + +1. Check out our contract repository, following [these instructions](../quick-start.md#cloning-the-repository-and-preparing-the-dependencies). You should then be able to run `yarn run truffle console --network xdai` which will connect you to the right network. You will need to be able to sign messages from the address in control of your CLNY (which will also be the address earning reputation for mining), which in most cases means pasting your private key into `truffle.js` before launching the console. For Ledger support, you can use `yarn run truffle console --network xdaiLedger`. For other hardware wallets, you will need to find an appropriate provider compatible with Truffle, and add it into `truffle.js` in your local version of the repository. + + An appropriate gas price for the current level of network use can be found at [https://blockscout.com/xdai/mainnet/](https://blockscout.com/xdai/mainnet/). The default value in `truffle.js` represent 2Gwei. + + +2. Create references to the various contracts that we will need. Run each of these commands in turn: ```javascript colonyNetwork = await IColonyNetwork.at("0x78163f593D1Fa151B4B7cacD146586aD2b686294" ); @@ -31,7 +57,7 @@ tokenLocking = await ITokenLocking.at(tokenLockingAddress); _Note that all of the following commands, where they represent a transaction being sent on-chain, are documented with `estimateGas`. This is deliberate, and so copying and pasting these instructions should not do anything on chain. Once you are happy with the command, and the call to `estimateGas` does not error, you remove the .`estimateGas` from the command and re-run it to execute it on-chain._ ::: -3\. Award the token locking contract the ability to take the CLNY you are intending to stake. The value used in this example is the minimum stake required; you may wish to stake more. +3. Award the token locking contract the ability to take the CLNY you are intending to stake. The value used in this example is the minimum stake required; you may wish to stake more. ```javascript // Approve the tokens @@ -50,7 +76,7 @@ await colonyNetwork.getMiningStake(accounts[0]); ] ``` -4\. Award delegated mining permissions to your 'disposable' address: +4. Award delegated mining permissions to your 'disposable' address: ```javascript const miningDelegate = "your-delegate-address" @@ -59,40 +85,194 @@ colonyNetwork.setMiningDelegate.estimateGas(miningDelegate, true) colonyNetwork.setMiningDelegate.estimateGas(miningDelegate, false) ``` -:::info -Using two addresses like this is not strictly required, but represents best practices - even if your mining process ended up compromised and the private key accessed, the worst that could be done with it would be to slowly have your stake slashed by misbehaving, which can be prevented by removing the permissions of the disposable address (see below) -::: - Congratulations! You've set up all the necessary permissions to run a miner. + + + + +### B. Getting a recent snapshot + +A recent snapshot, which should be from the last day or so, is available at [https://xdai.colony.io/reputation/xdai/latestState](https://xdai.colony.io/reputation/xdai/latestState). -#### C. Setting up the miner +After downloading, place it whichever directory you are running the reputation miner from, and rename it to `reputations.sqlite` (if you are using the commands above). Upon start, the miner will load this snapshot, and sync from there. Here's the command that downloads it and names it accordingly: -The biggest hurdle to running a reputation miner is syncing it initially. This requires an Xdai archive node. There is a public one available at [`https://xdai-archive.blockscout.com/`](https://xdai-archive.blockscout.com/), which we have successfully used in the past to sync a node from scratch, but can be unreliable for very old historical state. To speed up the syncing process, you can also use a recent snapshot of the reputation state tree ([see below](reputation-mining.md#snapshot)), but this doesn't remove the requirement for an archive node (for more recent historical state). +```bash +curl -o reputations.sqlite https://xdai.colony.io/reputation/xdai/latestState +# or +wget -O reputations.sqlite https://xdai.colony.io/reputation/xdai/latestState +``` + +### C. Setting up the miner + +The biggest hurdle to running a reputation miner is syncing it initially. This requires an Xdai archive node. Here's a list of providers that offer archive nodes on Gnosis chain (https://github.com/arddluma/awesome-list-rpc-nodes-providers#gnosis-xdai). You very likely will have to pay for that service. To speed up the syncing process, you can also use a recent snapshot of the reputation state tree ([see below](reputation-mining.md#snapshot)), but this doesn't remove the requirement for an archive node (for more recent historical state). Strictly speaking, once synced, an archive node is not required. However, should you fall behind (due to the miner not running for some reason), then you will need access to an archive mode to resume. -The most reliable way to run the miner is by using docker via the image provided by us, but you can also run it directly from a checked-out copy of our repository (which you already have, if you've completed the previous section). +The most reliable way to run the miner is by using Docker via the image provided by us, but you can also run it directly from a checked-out copy of our repository (which you already have, if you've completed the previous section). -Regardless of which you use, you will need the private key you wish mining transactions to be signed from. Putting the private key in an environment variable is recommended for security purposes - in the below examples, it could be placed in the appropriate variable with `export PRIVATE_KEY=0xdeadbeef00000000000000000deadbeef000000000000000000000000000dead` +Regardless of which you use, you will need the private key you wish mining transactions to be signed from. Putting the private key in an environment variable is recommended for security purposes. - -`docker run -it --env ARGS="--providerAddress https://xdai-archive.blockscout.com/" --env COLONYNETWORK_ADDRESS=0x78163f593D1Fa151B4B7cacD146586aD2b686294 --env SYNC_FROM_BLOCK="11897848" --env REPUTATION_JSON_PATH=/root/datadir/reputations.sqlite --env PRIVATE_KEY=$PRIVATE_KEY -v $(pwd):/root/datadir joincolony/reputation-miner:latest` + + +First create a directory for the miner data. The docker image will then create files in the directory you run this command from; you can alter the `-v` argument to change this behaviour. + +```bash +docker run --env REP_MINER_RPC_ENDPOINT="[YOUR_ARCHIVE_NODE_RPC_ADDRESS]" --env REP_MINER_PRIVATE_KEY="[YOUR_PRIVATE_KEY_FOR_MINING]" --env REP_MINER_DB_PATH="/root/datadir/reputations.sqlite" -p 3000:3000 -v $(pwd):/root/datadir joincolony/reputation-miner:latest +``` + - -`node ./packages/reputation-miner/bin/index.js --providerAddress https://xdai-archive.blockscout.com --colonyNetworkAddress 0x78163f593D1Fa151B4B7cacD146586aD2b686294 --syncFrom 11897847 --privateKey $PRIVATE_KEY --dbPath ./reputations.sqlite` + + +Note: this only works after you have successfully built the contracts + +```bash +node ./packages/reputation-miner/bin/index.js --rpcEndpoint [YOUR_ARCHIVE_NODE_RPC_ADDRESS] --privateKey [YOUR_PRIVATE_KEY_FOR_MINING] +``` + -The docker image will create files in the directory you run this command from; you can alter the `-v` argument to change this behaviour. +## Reputation Miner command line reference + +The reputation mining client can take various arguments that can be supplied via command line parameters or environment variables (useful when using Docker). You can aalways run + +```bash +node ./packages/reputation-miner/bin/index.js --help +``` + +to see all available options. + +### `--adapter` (`REP_MINER_ADAPTER`) + +Adapter to report mining logs to +[string] [choices: "console", "discord", "slack"] [default: "console"] + +### `--adapterLabel` (`REP_MINER_ADAPTER_LABEL`) + +Label for the adapter (only needed for Discord adapter) +[string] + +### `--auto` (`REP_MINER_AUTO`) + +Whether to automatically submit hashes and respond to challenges +[boolean] [default: true] + +The "auto" reputation mining client will: + +* Propose a new hash at the first possible block time, and submit until the maximum number has been reached (based on staked CLNY, with a maximum of 12 submissions allowed) +* Respond to challenges if there are disagreeing submissions. +* Confirm the last hash after the mining window closes and any disputes have been resolved. + +### `--colonyNetworkAddress` (`REP_MINER_COLONY_NETWORK_ADDRESS`) + +Ethereum address of the ColonyNetwork Smart Contract in the network the miner is connected to +[string] [default: "0x78163f593D1Fa151B4B7cacD146586aD2b686294"] -#### D. Getting a recent snapshot +The address of the Colony Network's `EtherRouter`. This is `0x78163f593D1Fa151B4B7cacD146586aD2b686294` for Gnosis Chain -A recent snapshot, which should be from the last day or so, is available at [https://xdai.colony.io/reputation/xdai/latestState](https://xdai.colony.io/reputation/xdai/latestState).\ -\ -After downloading, place it whichever directory you are running the reputation miner from, and rename it to `reputations.sqlite` (if you are using the commands above). Upon start, the miner will load this snapshot, and sync from there. +### `--dbPath` (`REP_MINER_DB_PATH`) -#### E. Rewards +Path where to save the database +[string] [default: "./reputations.sqlite"] + +### `--exitOnError` (`REP_MINER_EXIT_ON_ERROR`) + +Whether to exit when an error is hit or not. +[boolean] [default: false] + +### `--minerAddress` (`REP_MINER_MINER_ADDRESS`) (local development only) +Address of the miner account which the client will send reputation mining contract transactions from. Used when working with an unlocked account for the miner against **development networks only**. We provision twelve unlocked test accounts stored in `ganache-accounts.json` for testing that are available when starting a local ganache-cli instance via `npm run start:blockchain:client` command. + +### `--oracle` (`REP_MINER_ORACLE`) + +Whether to serve requests as a reputation oracle or not +[boolean] [default: true] + +### `--oraclePort` (`REP_MINER_ORACLE_PORT`) + +Port the reputation oracle should be exposed on. Only applicable if `oracle` is set to `true` +[number] [default: 3000] + +### `--privateKey` (`REP_MINER_PRIVATE_KEY`) + +The private key of the address that is mining, allowing the miner to sign transactions. +[string] + +Required for mining in production. + +### `--processingDelay` (`REP_MINER_PROCESSING_DELAY`) + +Delay between processing reputation logs (in blocks) +[number] [default: 10] + +### `--rpcEndpoint` (`REP_MINER_RPC_ENDPOINT`) + +http address of the RPC node to connect to. +[string] [default: "http://localhost:8545"] + +An archive node with RPC endpoint is required for mining in production. + +### `--syncFrom` (`REP_MINER_SYNC_FROM`) + +Block number to start reputation state sync from +[number] [default: 11897847] + +This is the block at which the reputation mining process was initialised. This number is static on Gnosis Chain: `11897847`. If you run into troubles when using this number, try `11897848`. + +Note that beginning the sync with a too-early block will result in an error. If you get this exception, try syncing from a more recent block. Note that the sync process can take long. Latest tests syncing a client from scratch to 28 reputation cycles took \~2 hours. + +## Rewards + +At the current time, there are no rewards for reputation mining yet. + +## Visualizations + +The reputation mining client comes with a set of built-in visualizers to make it easier to view reputation states and to see the current state of the mining process. Once a mining client is running and connected to a network, navigate to the client's address in a browser (i.e. `http://localhost:3000/`) to access the available visualization tools. + +## Get Reputation from the Reputation Oracle + +The reputation mining client will answer queries for reputation scores locally over HTTP. + +``` +http://127.0.0.1:3000/{reputationState}/{colonyAddress}/{skillId}/{userAddress} +``` + +An instance of the oracle is available for reputation queries against the Gnosis Chain network. + +``` +https://xdai.colony.io/reputation/{network}/{reputationState}/{colonyAddress}/{skillId}/{userAddress} +``` + +The oracle should be able to provide responses to any valid reputation score in all historical states, as well as the current state. For querying the colony-wide reputation instead of user-specific one, instead of {userAddress} use a zero address (`0x0000000000000000000000000000000000000000`) + +For example, you can get the reputation score of the miner in a reputation state `0xc7eb2cf60aa4848ce0feed5d713c07fd26e404dd50ca3b9e4f2fabef196ca3bc`) using the address of the Meta Colony (`0x14946533cefe742399e9734a123f0c02d0405a51`), the mining skill id (`2`), and address of a miner (`0x0A1d439C7d0b9244035d4F934BBF8A418B35d064`). + +``` +https://xdai.colony.io/reputation/mainnet/0xc7eb2cf60aa4848ce0feed5d713c07fd26e404dd50ca3b9e4f2fabef196ca3bc/0x14946533cefe742399e9734a123f0c02d0405a51/2/0x0A1d439C7d0b9244035d4F934BBF8A418B35d064 +``` + +The oracle returns + +``` +{"branchMask":"0xc000000000000000000000000000000000000000000000000000000000000000","siblings":["0x15c45d734bccc204df2e275d516250ed0a1cd60ccabadf49e2157a3e8067e59c","0xd4ee79473ec5573d706be030f3077c44aef06f26745349bbd93dcf5f4e254422"],"key":"0x14946533cefe742399e9734a123f0c02d0405a5100000000000000000000000000000000000000000000000000000000000000020a1d439c7d0b9244035d4f934bbf8a418b35d064","value":"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004","reputation":"0x0000000000000000000000000000000000000000000000000000000000000000","uid":"0x0000000000000000000000000000000000000000000000000000000000000004","reputationAmount":"0"} +``` + +## Using the Reputation Mining Client in development + +The client is set to provide a reputation update once every 24 hours. For testing, you'll likely want to "fast-forward" your network through a few submissions to see usable reputation. + +You can move the network forward by 24 hours with the following command. + +```bash +curl -H "Content-Type: application/json" -X POST --data '{"jsonrpc":"2.0","method":"evm_increaseTime","params":[86400],"id": 1}' localhost:8545 +``` + +Once you have moved the network forward 24 hours, you can then mine a new block with the following command. + +```bash +curl -H "Content-Type: application/json" -X POST --data '{"jsonrpc":"2.0","method":"evm_mine","params":[]}' localhost:8545 +``` -At the current time, there are no rewards for reputation mining, but this should change in the coming weeks. +Note that because reputation is awarded for the _previous_ submission window, you will need to use the "fast-forward" command above to speed through at least 2 reputation updates before noticing a change in the miner's reputation. diff --git a/packages/reputation-miner/Dockerfile b/packages/reputation-miner/Dockerfile index 1ab9630f07..3330047c3d 100644 --- a/packages/reputation-miner/Dockerfile +++ b/packages/reputation-miner/Dockerfile @@ -1,11 +1,8 @@ -FROM node:14-bullseye -RUN apt-get update || : && apt-get install python -y +FROM node:16.20.2-slim +COPY ./build ./build COPY ./packages ./packages COPY ./package.json ./ COPY ./package-lock.json ./ -COPY ./build ./build -RUN npm install -RUN cd ./packages/reputation-miner/ && npm i -RUN cd ./packages/package-utils/ && npm i +RUN npm ci EXPOSE 3000 -CMD node $NODE_ARGS packages/reputation-miner/bin/index.js --dbPath $REPUTATION_JSON_PATH --colonyNetworkAddress $COLONYNETWORK_ADDRESS --privateKey $PRIVATE_KEY --syncFrom $SYNC_FROM_BLOCK $ARGS +CMD node $NODE_ARGS packages/reputation-miner/bin/index.js diff --git a/packages/reputation-miner/ReputationMiner.js b/packages/reputation-miner/ReputationMiner.js index ec76812b68..4d91e2466d 100644 --- a/packages/reputation-miner/ReputationMiner.js +++ b/packages/reputation-miner/ReputationMiner.js @@ -16,6 +16,8 @@ const minStake = ethers.BigNumber.from(10).pow(18).mul(2000); const DAY_IN_SECONDS = 60 * 60 * 24; +const BLOCK_PAGING_SIZE = 1000000; + class ReputationMiner { /** * Constructor for ReputationMiner @@ -1359,10 +1361,17 @@ class ReputationMiner { if (!blockNumber) { throw new Error("Block number not supplied to sync"); } - // Get the events + let events = []; + const latestBlockNumber = await this.realProvider.getBlockNumber(); + const filter = this.colonyNetwork.filters.ReputationMiningCycleComplete(null, null); - filter.fromBlock = blockNumber; - const events = await this.realProvider.getLogs(filter); + filter.toBlock = Math.max(blockNumber - 1, 0); + while (filter.toBlock !== latestBlockNumber) { + filter.fromBlock = filter.toBlock + 1; + filter.toBlock = Math.min(filter.fromBlock + BLOCK_PAGING_SIZE, latestBlockNumber); + const partialEvents = await this.realProvider.getLogs(filter); + events = events.concat(partialEvents); + } let localHash = await this.reputationTree.getRootHash(); let applyLogs = false; @@ -1428,6 +1437,7 @@ class ReputationMiner { // Some more cycles might have completed since we started syncing const lastEventBlock = events[events.length - 1].blockNumber filter.fromBlock = lastEventBlock; + filter.toBlock = "latest"; const sinceEvents = await this.realProvider.getLogs(filter); if (sinceEvents.length > 1){ console.log("Some more cycles have completed during the sync process. Continuing to sync...") @@ -1558,10 +1568,13 @@ class ReputationMiner { await this.instantiateJustificationTree(jtType); try { - const justificationHashFile = await fs.readFile(this.justificationCachePath, 'utf8') this.justificationHashes = JSON.parse(justificationHashFile); + } catch (err) { + console.log('Could not find justificationTreeCache.json. It will be created...') + } + try { for (let i = 0; i < Object.keys(this.justificationHashes).length; i += 1) { const hash = Object.keys(this.justificationHashes)[i]; const tx = await this.justificationTree.insert( diff --git a/packages/reputation-miner/ReputationMinerClient.js b/packages/reputation-miner/ReputationMinerClient.js index 22954dbe2a..3cca864100 100644 --- a/packages/reputation-miner/ReputationMinerClient.js +++ b/packages/reputation-miner/ReputationMinerClient.js @@ -47,6 +47,7 @@ class ReputationMinerClient { * @param {bool} oracle Whether to serve requests as a reputation oracle or not * @param {bool} exitOnError Whether to exit when an error is hit or not. * @param {Object} adapter An object with .log and .error that controls where the output from the miner ends up. + * @param {Number} processingDelay Delay between processing reputation logs (in blocks) */ constructor({ minerAddress, loader, realProviderPort, oraclePort = 3000, privateKey, provider, useJsTree, dbPath, auto, oracle, exitOnError, adapter, processingDelay }) { // eslint-disable-line max-len this._loader = loader; @@ -749,7 +750,7 @@ class ReputationMinerClient { // Submit hash let submitRootHashTx = await this._miner.submitRootHash(entryIndex); - if (!submitRootHashTx.nonce) { + if (!Object.prototype.hasOwnProperty.call(submitRootHashTx, "nonce")) { // Assume we've been given back the submitRootHashTx hash. submitRootHashTx = await this._miner.realProvider.getTransaction(submitRootHashTx); } diff --git a/packages/reputation-miner/bin/index.js b/packages/reputation-miner/bin/index.js index 0d18e3b344..fb02b1dbcf 100644 --- a/packages/reputation-miner/bin/index.js +++ b/packages/reputation-miner/bin/index.js @@ -1,9 +1,5 @@ const path = require("path"); -const { argv } = require("yargs") - .option('privateKey', {string:true}) - .option('colonyNetworkAddress', {string:true}) - .option('minerAddress', {string:true}) - .option('providerAddress', {type: "array", default: []}); +const yargs = require("yargs") const ethers = require("ethers"); const backoff = require("exponential-backoff").backOff; @@ -11,33 +7,102 @@ const ReputationMinerClient = require("../ReputationMinerClient"); const { ConsoleAdapter, SlackAdapter, DiscordAdapter, TruffleLoader } = require("../../package-utils"); -const supportedInfuraNetworks = ["goerli", "rinkeby", "ropsten", "kovan", "mainnet"]; +const { argv } = yargs + .option("adapter", { + describe: "Adapter to report mining logs to", + type: "string", + choices: ["console", "discord", "slack"], + default: "console" + }) + .option("adapterLabel", { + describe: "Label for the adapter (only needed for Discord adapter)", + type: "string", + }) + .option("auto", { + describe: "Whether to automatically submit hashes and respond to challenges", + type: "boolean", + default: true + }) + .option("colonyNetworkAddress", { + describe: "Ethereum address of the ColonyNetwork Smart Contract in the network the Miner is connected to", + type: "string", + default: "0x78163f593D1Fa151B4B7cacD146586aD2b686294" + }) + .option("dbPath", { + describe: "Path where to save the database", + type: "string", + default: "./reputations.sqlite" + }) + .option("exitOnError", { + describe: "Whether to exit when an error is hit or not.", + type: "boolean", + default: false, + }) + .option("minerAddress", { + // eslint-disable-next-line max-len + describe: "Address of the miner account which the client will send reputation mining contract transactions from. Used when working with an unlocked account for the miner against development networks only", + type: "string", + conflicts: "privateKey", + hidden: true + }) + .option("oracle", { + describe: "Whether to serve requests as a reputation oracle or not", + type: "boolean", + default: true + }) + .option("oraclePort", { + describe: "Port the reputation oracle should be exposed on. Only applicable if `oracle` is set to `true`", + type: "number", + default: 3000, + }) + .option("privateKey", { + // eslint-disable-next-line max-len + describe: "The private key of the address that is mining, allowing the miner to sign transactions.", + type: "string", + conflicts: "minerAddress", + }) + .option("processingDelay", { + describe: "Delay between processing reputation logs (in blocks)", + type: "number", + default: 10, + }) + .option("rpcEndpoint", { + describe: "http address of the RPC node to connect to", + type: "string", + default: "http://localhost:8545", + }) + .option("syncFrom", { + describe: "Block number to start reputation state sync from", + type: "number", + default: 11897847, + }) + .env("REP_MINER") +; + const { - minerAddress, - privateKey, + adapter, + adapterLabel, + auto, colonyNetworkAddress, dbPath, - network, - localPort, - localProviderAddress, - providerAddress, - syncFrom, - auto, - oracle, exitOnError, - adapter, + minerAddress, + oracle, oraclePort, + privateKey, processingDelay, - adapterLabel, + rpcEndpoint, + syncFrom, } = argv; + class RetryProvider extends ethers.providers.StaticJsonRpcProvider { constructor(connectionInfo, adapterObject){ super(connectionInfo); this.adapter = adapterObject; } - static attemptCheck(err, attemptNumber){ + static attemptCheck(_err, attemptNumber){ console.log("Retrying RPC request #", attemptNumber); if (attemptNumber === 5){ return false; @@ -46,28 +111,27 @@ class RetryProvider extends ethers.providers.StaticJsonRpcProvider { } getNetwork(){ - return backoff(() => super.getNetwork(), {retry: RetryProvider.attemptCheck}); + return backoff(() => super.getNetwork(), { retry: RetryProvider.attemptCheck }); } // This should return a Promise (and may throw erros) // method is the method name (e.g. getBalance) and params is an // object with normalized values passed in, depending on the method perform(method, params) { - return backoff(() => super.perform(method, params), {retry: RetryProvider.attemptCheck, startingDelay: 1000}); + return backoff(() => super.perform(method, params), { retry: RetryProvider.attemptCheck, startingDelay: 1000 }); } } -if ((!minerAddress && !privateKey) || !colonyNetworkAddress || !syncFrom) { - console.log("❗️ You have to specify all of ( --minerAddress or --privateKey ) and --colonyNetworkAddress and --syncFrom on the command line!"); +if (!minerAddress && !privateKey) { + console.log("❗️ You have to specify --privateKey (or --minerAddress when in development mode)"); process.exit(); } - let adapterObject; -if (adapter === 'slack') { +if (adapter === "slack") { adapterObject = new SlackAdapter(); -} else if (adapter === 'discord'){ +} else if (adapter === "discord"){ adapterObject = new DiscordAdapter(adapterLabel); } else { adapterObject = new ConsoleAdapter(); @@ -77,35 +141,33 @@ const loader = new TruffleLoader({ contractDir: path.resolve(__dirname, "..", "..", "..", "build", "contracts") }); -let provider; -if (network) { - if (!supportedInfuraNetworks.includes(network)) { - console.log(`❗️ "network" option accepts only supported Infura networks: ${supportedInfuraNetworks} !`); - process.exit(); - } - provider = new ethers.providers.InfuraProvider(network); -} else if (providerAddress.length === 0){ - const rpcEndpoint = `${localProviderAddress || "http://localhost"}:${localPort || "8545"}`; - provider = new ethers.providers.JsonRpcProvider(rpcEndpoint); -} else { - const providers = providerAddress.map(endpoint => { - const {protocol, username, password, host, pathname} = new URL(endpoint); - const connectionInfo = { - url: `${protocol}//${host}${pathname}`, - user: decodeURI(username), - password: decodeURI(password.replace(/%23/, '#')), - } - return new RetryProvider(connectionInfo, adapterObject); - }) - // This is, at best, a huge hack... - providers.forEach(x => x.getNetwork()); - - // The Fallback provider somehow strips out blockTag, so isn't suitable for use during syncing. - // See https://github.com/ethers-io/ethers.js/discussions/1960 - // When sorted, use this line instead. - // provider = new ethers.providers.FallbackProvider(providers, 1) - [ provider ] = providers; -} +const { protocol, username, password, host, pathname } = new URL(rpcEndpoint); +const connectionInfo = { + url: `${protocol}//${host}${pathname}`, + user: decodeURI(username), + password: decodeURI(password.replace(/%23/, "#")), +}; +const provider = new RetryProvider(connectionInfo, adapterObject); + +console.log(` +----------------------------------------------------------------------------------------- +Running with the following configuration: + adapter: ${adapter} + adapterLabel: ${adapterLabel} + auto: ${auto} + colonyNetworkAddress: ${colonyNetworkAddress} + dbPath: ${dbPath} + exitOnError: ${exitOnError} + minerAddress: ${minerAddress} + oracle: ${oracle} + oraclePort: ${oraclePort} + privateKey: --REDACTED--, + processingDelay: ${processingDelay} + rpcEndpoint: ${rpcEndpoint} + syncFrom: ${syncFrom} + +----------------------------------------------------------------------------------------- +`) const client = new ReputationMinerClient({ loader, diff --git a/packages/reputation-miner/package.json b/packages/reputation-miner/package.json index 76772be62b..0e90867fab 100644 --- a/packages/reputation-miner/package.json +++ b/packages/reputation-miner/package.json @@ -1,7 +1,11 @@ { "name": "@colony/reputation-miner", "version": "0.1.0", - "description": "Colony", + "description": "Colony Reputation Miner", + "engines": { + "node": "^14.18 || ^16.20", + "npm": "^8.19" + }, "main": "ReputationMinerClient.js", "directories": { "test": "test" diff --git a/test/reputation-system/client-sync-functionality.js b/test/reputation-system/client-sync-functionality.js index 2d8d5f5724..16f02660b8 100644 --- a/test/reputation-system/client-sync-functionality.js +++ b/test/reputation-system/client-sync-functionality.js @@ -262,8 +262,10 @@ process.env.SOLIDITY_COVERAGE useJsTree: true, dbPath: fileName, }); + await reputationMiner3.initialise(colonyNetwork.address); - await reputationMiner3.sync("latest"); + const latestBlock = await currentBlock(); + await reputationMiner3.sync(parseInt(latestBlock.number, 10)); const loadedState = await reputationMiner3.getRootHash(); expect(loadedState).to.equal(currentState);