diff --git a/.github/actions/terraform/Dockerfile b/.github/actions/terraform/Dockerfile deleted file mode 100644 index d620894fd9..0000000000 --- a/.github/actions/terraform/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -FROM hashicorp/terraform:0.15.0 - -RUN apk add bash - -COPY main.sh /terraform/main.sh -COPY apply.sh /terraform/apply.sh -COPY cleanup.sh /terraform/cleanup.sh - -ENTRYPOINT [ "/terraform/main.sh" ] diff --git a/.github/actions/terraform/README.md b/.github/actions/terraform/README.md deleted file mode 100644 index 10a9d867de..0000000000 --- a/.github/actions/terraform/README.md +++ /dev/null @@ -1,47 +0,0 @@ -# Terraform - -Suppose you have a job that applies AWS resources, runs your tests on these resources, and finally, destroys them whatever succeeded, failed or aborted. You can do this safely by writing Terraform configuration files and passing the directory path. - -This action use [`post`](https://github.com/actions/runner/blob/be9632302ceef50bfb36ea998cea9c94c75e5d4d/docs/adrs/0361-wrapper-action.md) keyword to ensure that clean work always be done. - -# Usage - - -```yaml -- uses: ./.github/actions/terraform - with: - # Required, directory path to Terraform configuration files - terraform_dir: /path/to/terraform/configuration/directory -``` - - -# Examples - -```yaml -# Generate a random SSH key pair to access instances, such as Ansible, via SSH -- run: ssh-keygen -N "" -f ${{ env.PRIVATE_KEY_PATH }} - -- name: Apply Resources - uses: ./.github/actions/terraform - env: - # You may declare Terraform variables in `variables.tf`. - # - # Setting declared variables as environment variables is a common practice. - # - # [Learn more about Terraform variables](https://www.terraform.io/docs/language/values/variables.html#environment-variables) - TF_VAR_access_key: ${{ secrets.AWS_ACCESS_KEY }} - TF_VAR_secret_key: ${{ secrets.AWS_SECRET_KEY }} - TF_VAR_private_key_path: ${{ env.PRIVATE_KEY_PATH }} - TF_VAR_public_key_path: ${{ env.PUBLIC_KEY_PATH }} - TF_VAR_instance_type: t3.micro - with: - terraform_dir: /path/to/terraform/configuration/directory - -- name: Output - working-directory: /path/to/terraform/configuration/directory - run: | - terraform output -raw "" >> /path/to/ansible/inventory-file - -# Ansible-playbook do tests -- run: ansible-playbook playbook.yml -i /path/to/ansible/inventory-file ... -``` diff --git a/.github/actions/terraform/action.yml b/.github/actions/terraform/action.yml deleted file mode 100644 index 1d710361b4..0000000000 --- a/.github/actions/terraform/action.yml +++ /dev/null @@ -1,14 +0,0 @@ -name: 'Terraform' - -description: 'Apply Terraform defined Infrastructures on-demand, destroy at job end' - -inputs: - terraform_dir: - description: 'Directory path to Terraform configuration files' - required: true - -runs: - using: 'docker' - image: 'Dockerfile' - entrypoint: '/terraform/apply.sh' - post-entrypoint: '/terraform/cleanup.sh' diff --git a/.github/actions/terraform/apply.sh b/.github/actions/terraform/apply.sh deleted file mode 100755 index 95a553b370..0000000000 --- a/.github/actions/terraform/apply.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -# Call by action `entrypoint` - -SCRIPT_PATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" -${SCRIPT_PATH}/main.sh apply diff --git a/.github/actions/terraform/cleanup.sh b/.github/actions/terraform/cleanup.sh deleted file mode 100755 index 967733bc43..0000000000 --- a/.github/actions/terraform/cleanup.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -# Call by action `post_entrypoint` - -SCRIPT_PATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" -${SCRIPT_PATH}/main.sh cleanup diff --git a/.github/actions/terraform/main.sh b/.github/actions/terraform/main.sh deleted file mode 100755 index acad5d38d8..0000000000 --- a/.github/actions/terraform/main.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -# Path to location of terraform files -TERRAFORM_DIR=${INPUT_TERRAFORM_DIR:-${TERRAFORM_DIR}} - -function apply_infrastructure () { - cd ${TERRAFORM_DIR} - terraform init - terraform plan - terraform apply -auto-approve -} - -function cleanup_infrastructure () { - cd ${TERRAFORM_DIR} - terraform init - terraform destroy -auto-approve -} - -function cleanup_terraform_footprint () { - rm -rf ${TERRAFORM_DIR}/.terraform - rm -rf ${TERRAFORM_DIR}/terraform.tfstate -} - -function main () { - [ $1 ] || { echo "Wrong usage"; exit 1; } - - local command="${1}" - shift 1 - case ${command} in - apply ) - apply_infrastructure - ;; - cleanup ) - cleanup_infrastructure - cleanup_terraform_footprint - ;; - * ) - echo "Wrong usage"; exit 1; - ;; - esac -} - -main "$@" diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 37736a60e7..604e9b1b61 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -4,123 +4,39 @@ on: # Submit your review with a comment body containing "#benchmark" pull_request_review: types: [ submitted ] + push: + branches: ['benchmark_update'] schedule: - cron: '0 0 * * *' # Manurally trigger workflow_dispatch: jobs: - build_modified_ckb: - name: Build Modified CKB - runs-on: ubuntu-latest - if: | - ( - github.event_name == 'pull_request_review' && - contains(github.event.review.body, '#benchmark') - ) || ( - github.event_name == 'schedule' && - github.repository_owner == 'nervosnetwork' - ) || ( - github.event_name == 'workflow_dispatch' - ) - env: - CARGO_TARGET_DIR: "${{ github.workspace }}/../target" - steps: - - uses: actions/checkout@v2 - - name: Modify Consensus Parameters And Build CKB - run: | - sed -i 's/const TWO_IN_TWO_OUT_COUNT: u64 = .*;$/const TWO_IN_TWO_OUT_COUNT: u64 = 8_000;/g' spec/src/consensus.rs - sed -i 's/const MAX_BLOCK_PROPOSALS_LIMIT: u64 = .*;$/const MAX_BLOCK_PROPOSALS_LIMIT: u64 = 12_000;/g' spec/src/consensus.rs - make build - cd ${{ env.CARGO_TARGET_DIR }}/release - tar cfJ ckb.tar.xz ckb - mv ckb.tar.xz ${{ github.workspace }} - - uses: actions/upload-artifact@v2 - with: - name: ckb.tar.xz - path: ckb.tar.xz benchmark: name: Benchmark runs-on: ubuntu-latest - needs: [ build_modified_ckb ] + # if: | + # ( + # github.event_name == 'pull_request_review' && + # contains(github.event.review.body, '#benchmark') + # ) || ( + # github.event_name == 'schedule' && + # github.repository_owner == 'nervosnetwork' + # ) || ( + # github.event_name == 'workflow_dispatch' + # ) env: - TERRAFORM_DIR: ${{ github.workspace }}/.github/workflows/benchmark/terraform - ANSIBLE_DIR: ${{ github.workspace }}/.github/workflows/benchmark/ansible - ANSIBLE_INVENTORY: ${{ github.workspace }}/.github/workflows/benchmark/ansible/inventory.yml - PRIVATE_KEY_PATH: ${{ github.workspace }}/id_rsa - PUBLIC_KEY_PATH: ${{ github.workspace }}/id_rsa.pub + AWS_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY }} + AWS_SECRET_KEY: ${{ secrets.AWS_SECRET_KEY }} + AWS_EC2_TYPE: "c5.xlarge" + GITHUB_TOKEN: ${{ secrets.REPO_ACCESS_TOKEN }} steps: - uses: actions/checkout@v2 - - name: Prepare - Download CKB Tarball - uses: actions/download-artifact@v2 - with: - name: ckb.tar.xz - - # Prepare - - name: Prepare - Generate Random SSH Key - run: ssh-keygen -N "" -f ${{ env.PRIVATE_KEY_PATH }} - - name: Prepare - Apply Resources Based on Terraform Files - uses: ./.github/actions/terraform - env: - # Environment variables used inside terraform/variables.tf - TF_VAR_access_key: ${{ secrets.AWS_ACCESS_KEY }} - TF_VAR_secret_key: ${{ secrets.AWS_SECRET_KEY }} - TF_VAR_prefix: benchmark-${{ github.repository }}-${{ github.run_id }} - TF_VAR_private_key_path: ${{ env.PRIVATE_KEY_PATH }} - TF_VAR_public_key_path: ${{ env.PUBLIC_KEY_PATH }} - with: - terraform_dir: ${{ env.TERRAFORM_DIR }} - - name: Prepare - Output Ansible Inventory Based on Terraform State - working-directory: ${{ env.TERRAFORM_DIR }} + - name: Benchmark run: | - # `ansible_hosts` is defined in terraform/main.tf - terraform output -raw ansible_hosts > ${{ env.ANSIBLE_INVENTORY }} - terraform output -raw ansible_hosts - - # Run - - name: Run Ansible Playbook - shell: bash - working-directory: ${{ env.ANSIBLE_DIR }} - env: - ANSIBLE_PRIVATE_KEY_FILE: ${{ env.PRIVATE_KEY_PATH }} - run: | - ansible-galaxy install -r requirements.yml --force - - # Install CKB on group instance - ansible-playbook playbook.yml \ - -e 'hostname=instances' \ - -e 'ckb_local_source=${{ github.workspace }}/ckb.tar.xz' \ - -t ckb_install,ckb_configure - ansible-playbook playbook.yml -e 'hostname=instances' -t ckb_start - - ansible-playbook playbook.yml -e 'hostname=bastions' -t ckb_benchmark_install - ansible-playbook playbook.yml -e 'hostname=bastions' -t ckb_benchmark_prepare - ansible-playbook playbook.yml -e 'hostname=bastions' -t ckb_benchmark_start - ansible-playbook playbook.yml -e 'hostname=bastions' -t process_result - - - name: Post Run - Construct Report - run: | - echo 'BENCHMARK_REPORT<> $GITHUB_ENV - cat ${ANSIBLE_DIR}/report.yml >> $GITHUB_ENV - echo 'EOF' >> $GITHUB_ENV - - if [ ${{ github.event_name }} = 'pull_request_review' ] ; then - echo 'ISSUE_NUMBER=${{ github.event.pull_request.number }}' >> $GITHUB_ENV - else - echo 'ISSUE_NUMBER=2372' >> $GITHUB_ENV - fi - - name: Post Run - Comment Report - uses: peter-evans/create-or-update-comment@v1 - with: - issue-number: ${{ env.ISSUE_NUMBER }} - body: | - **Benchmark Report**: https://www.github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} - - ```yaml - ${{ env.BENCHMARK_REPORT }} - ``` - continue-on-error: true + git clone https://github.com/nervosnetwork/ckb-integration-test.git + devtools/ci/benchmark.sh - uses: actions/upload-artifact@v2 with: name: report.yml diff --git a/.github/workflows/benchmark/ansible/README.md b/.github/workflows/benchmark/ansible/README.md deleted file mode 100644 index 5b33122632..0000000000 --- a/.github/workflows/benchmark/ansible/README.md +++ /dev/null @@ -1,55 +0,0 @@ -# Ansible Playbook Used For Benchmark Workflow - -This Ansible playbook is part of the "Benchmark" workflow. - -## Pre-Install - -``` -ansible-galaxy install -r requirements.yml -``` - -## Inventory File - -To use this playbook, the provided inventory file must have 2 groups of hosts: - * Group "instances" indicates CKB nodes and hosts named by "instance-0", "instance-1" and so on. - - * Group "bastions" indicates CKB-Benchmark nodes, only have one host, named "bastion-0" - -All hosts must have `instance_type` variables, which are used to generate the benchmark report. - -Here is an example: - -```yaml -instances: - hosts: - instance-0: - ansible_user: ubuntu - ansible_host: 1.23.45.123 - instance_type: c5.large - instance-0: - ansible_user: ubuntu - ansible_host: 1.23.45.124 - instance_type: c5.large - -bastions: - hosts: - bastion-0: - ansible_user: ubuntu - ansible_host: 1.23.45.125 - instance_type: t2.large -``` - -## Usage Example - -```bash -export ANSIBLE_INVENTORY= -export ANSIBLE_PRIVATE_KEY_FILE= - -ansible-playbook playbook.yml -e 'hostname=instances' -t ckb_install,ckb_configure -ansible-playbook playbook.yml -e 'hostname=instances' -t ckb_start - -ansible-playbook playbook.yml -e 'hostname=bastions' -t ckb_benchmark_install -ansible-playbook playbook.yml -e 'hostname=bastions' -t ckb_benchmark_prepare -ansible-playbook playbook.yml -e 'hostname=bastions' -t ckb_benchmark_start -ansible-playbook playbook.yml -e 'hostname=bastions' -t process_result -``` diff --git a/.github/workflows/benchmark/ansible/ansible.cfg b/.github/workflows/benchmark/ansible/ansible.cfg deleted file mode 100644 index 09e07747a2..0000000000 --- a/.github/workflows/benchmark/ansible/ansible.cfg +++ /dev/null @@ -1,4 +0,0 @@ -[defaults] -host_key_checking = False -gathering = explicit -timeout = 60 diff --git a/.github/workflows/benchmark/ansible/files/QmUpChEcQwBcnhTqoNXxynW2B4hNq9R2USgWZKHUnkWdGd b/.github/workflows/benchmark/ansible/files/QmUpChEcQwBcnhTqoNXxynW2B4hNq9R2USgWZKHUnkWdGd deleted file mode 100644 index 6d3bcae468..0000000000 --- a/.github/workflows/benchmark/ansible/files/QmUpChEcQwBcnhTqoNXxynW2B4hNq9R2USgWZKHUnkWdGd +++ /dev/null @@ -1 +0,0 @@ -mb\7JS_C&8Pׂ] \ No newline at end of file diff --git a/.github/workflows/benchmark/ansible/files/QmcjX7xQxHJ2spbpb8uCD89LiCNbFsnFCLBA1Mu9SKgXjo b/.github/workflows/benchmark/ansible/files/QmcjX7xQxHJ2spbpb8uCD89LiCNbFsnFCLBA1Mu9SKgXjo deleted file mode 100644 index 0756be230f..0000000000 Binary files a/.github/workflows/benchmark/ansible/files/QmcjX7xQxHJ2spbpb8uCD89LiCNbFsnFCLBA1Mu9SKgXjo and /dev/null differ diff --git a/.github/workflows/benchmark/ansible/files/Qmctiy6jRrpZnf7SCWjCskwetwKroW7bhZJc28PydsBevX b/.github/workflows/benchmark/ansible/files/Qmctiy6jRrpZnf7SCWjCskwetwKroW7bhZJc28PydsBevX deleted file mode 100644 index 99cc5060d5..0000000000 --- a/.github/workflows/benchmark/ansible/files/Qmctiy6jRrpZnf7SCWjCskwetwKroW7bhZJc28PydsBevX +++ /dev/null @@ -1 +0,0 @@ -`&{R}m@g G1t\6Yl \ No newline at end of file diff --git a/.github/workflows/benchmark/ansible/files/benchmark-spec.toml b/.github/workflows/benchmark/ansible/files/benchmark-spec.toml deleted file mode 100644 index e629505ef6..0000000000 --- a/.github/workflows/benchmark/ansible/files/benchmark-spec.toml +++ /dev/null @@ -1,97 +0,0 @@ -name = "ckb_dev" - -[genesis] -version = 0 -parent_hash = "0x0000000000000000000000000000000000000000000000000000000000000000" -timestamp = 0 -compact_target = 0x20010000 -uncles_hash = "0x0000000000000000000000000000000000000000000000000000000000000000" -nonce = "0x0" - -[genesis.genesis_cell] -message = "ckb_dev" - -[genesis.genesis_cell.lock] -code_hash = "0x0000000000000000000000000000000000000000000000000000000000000000" -args = "0x" -hash_type = "data" - -# An array list paths to system cell files, which is absolute or relative to -# the directory containing this config file. -[[genesis.system_cells]] -file = { bundled = "specs/cells/secp256k1_blake160_sighash_all" } -create_type_id = true -capacity = 100_000_0000_0000 -[[genesis.system_cells]] -file = { bundled = "specs/cells/dao" } -create_type_id = true -capacity = 16_000_0000_0000 -[[genesis.system_cells]] -file = { bundled = "specs/cells/secp256k1_data" } -create_type_id = false -capacity = 1_048_617_0000_0000 -[[genesis.system_cells]] -file = { bundled = "specs/cells/secp256k1_blake160_multisig_all" } -create_type_id = true -capacity = 100_000_0000_0000 - -[genesis.system_cells_lock] -code_hash = "0x0000000000000000000000000000000000000000000000000000000000000000" -args = "0x" -hash_type = "data" - -# Dep group cells -[[genesis.dep_groups]] -name = "secp256k1_blake160_sighash_all" -files = [ - { bundled = "specs/cells/secp256k1_data" }, - { bundled = "specs/cells/secp256k1_blake160_sighash_all" }, -] -[[genesis.dep_groups]] -name = "secp256k1_blake160_multisig_all" -files = [ - { bundled = "specs/cells/secp256k1_data" }, - { bundled = "specs/cells/secp256k1_blake160_multisig_all" }, -] - -# For first 11 block -[genesis.bootstrap_lock] -code_hash = "0x0000000000000000000000000000000000000000000000000000000000000000" -args = "0x" -hash_type = "type" - -# Burn -[[genesis.issued_cells]] -capacity = 8_400_000_000_00000000 -lock.code_hash = "0x0000000000000000000000000000000000000000000000000000000000000000" -lock.args = "0x62e907b15cbf27d5425399ebf6f0fb50ebb88f18" -lock.hash_type = "data" - -# issue for random generated private key: d00c06bfd800d27397002dca6fb0993d5ba6399b4238b2f29ee9deb97593d2bc -[[genesis.issued_cells]] -capacity = 20_000_000_000_00000000 -lock.code_hash = "0x9bd7e06f3ecf4be0f2fcd2188b23f1b9fcc88e5d4b65a8637b17723bbda3cce8" -lock.args = "0xc8328aabcd9b9e8e64fbc566c4385c3bdeb219d7" -lock.hash_type = "type" - -# issue for random generated private key: 63d86723e08f0f813a36ce6aa123bb2289d90680ae1e99d4de8cdb334553f24d -[[genesis.issued_cells]] -capacity = 5_198_735_037_00000000 -lock.code_hash = "0x9bd7e06f3ecf4be0f2fcd2188b23f1b9fcc88e5d4b65a8637b17723bbda3cce8" -lock.args = "0x470dcdc5e44064909650113a274b3b36aecb6dc7" -lock.hash_type = "type" - -[params] -initial_primary_epoch_reward = 11_917_808_219178080 -secondary_epoch_reward = 613_698_63013698 -max_block_cycles = 100_000_000_000 -cellbase_maturity = 0 -primary_epoch_reward_halving_interval = 8760 -epoch_duration_target = 14400 -genesis_epoch_length = 1000 -# For development and testing purposes only. -# Keep difficulty be permanent if the pow is Dummy. (default: false) -# permanent_difficulty_in_dummy = true - -[pow] -func = "Dummy" diff --git a/.github/workflows/benchmark/ansible/files/process-metrics.py b/.github/workflows/benchmark/ansible/files/process-metrics.py deleted file mode 100755 index 3582767f0a..0000000000 --- a/.github/workflows/benchmark/ansible/files/process-metrics.py +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env python3 - -# Usage: `./process-metrics.py metrics.json` - -import os -import sys -import json -import yaml - -INSTANCE_TYPE = os.getenv("INSTANCE_TYPE") -INSTANCE_BASTION_TYPE = os.getenv("INSTANCE_BASTION_TYPE") - - -def rewrite_metric(metric): - # Construct new metric - metric["instance_type"] = INSTANCE_TYPE - metric["instance_bastion_type"] = INSTANCE_BASTION_TYPE - return metric - - -def main(): - metrics = [] - for line in open(sys.argv[1]): - metric = json.loads(line) - metric = rewrite_metric(metric) - metrics.append(metric) - - print(yaml.dump(metrics)) - - -main() diff --git a/.github/workflows/benchmark/ansible/playbook.yml b/.github/workflows/benchmark/ansible/playbook.yml deleted file mode 100644 index dd19bd988b..0000000000 --- a/.github/workflows/benchmark/ansible/playbook.yml +++ /dev/null @@ -1,80 +0,0 @@ ---- -- hosts: "{{ hostname }}" - gather_facts: yes - tasks: - - include_vars: vars/all.yml - tags: [ always ] - - name: Operate CKB Via Ansible-CKB - include_role: - name: ansible-ckb - public: false - tags: - - ckb_install - - ckb_configure - - ckb_restart - - ckb_start - - ckb_stop - - ckb_status - - ckb_miner_restart - - ckb_miner_start - - ckb_miner_stop - - - name: Install And Configure CKB Benchmark Via Ansible-CKB-Benchmark - include_role: - name: ansible-ckb-benchmark - public: false - tags: - - ckb_benchmark_setup - - ckb_benchmark_install - - - name: Set Facts - `ckb_urls` - block: - - name: Set Facts - Declare `ckb_urls` - set_fact: - ckb_urls: [] - - name: Set Facts - Extend `ckb_urls` - vars: - ckb_port: "{{ hostvars[item].ckb_rpc_listen_address | default(ckb_rpc_listen_address) | regex_replace(\".*:\", \"\")}}" - ckb_host: "{{ hostvars[item].ansible_host }}" - ckb_url: "http://{{ ckb_host }}:{{ ckb_port }}" - set_fact: - ckb_urls: "{{ ckb_urls + [ ckb_url ] }}" - with_items: "{{ groups.instances }}" - tags: - - ckb_benchmark_prepare - - ckb_benchmark_start - - # NOTE: It seems bug that when uses `include_role` inside block statement, - # tags cannot pass through properly. So please do not put - # `include_role` inside block statement. - - name: Start CKB Benchmark - vars: - ckb_benchmark_rpc_urls: "{{ ckb_urls }}" - include_role: - name: ansible-ckb-benchmark - public: false - tags: - - ckb_benchmark_prepare - - ckb_benchmark_start - - - name: Fetch CKB Benchmark Logfiles - become: true - fetch: - flat: true - src: "{{ item.src }}" - dest: "{{ item.dest }}" - with_items: - - { src: "/var/lib/ckb-benchmark/data/ckb-bench.json", dest: "ckb-bench.json" } - - { src: "/var/lib/ckb-benchmark/data/ckb-bench.log", dest: "ckb-bench.log" } - tags: - - process_result - - - name: Process TPS-Bench Result `metrics.yml` - run_once: true - delegate_to: localhost - environment: - INSTANCE_TYPE: "{{ hostvars['instance-0'].instance_type }}" - INSTANCE_BASTION_TYPE: "{{ hostvars['bastion-0'].instance_type }}" - shell: ./files/process-metrics.py ckb-bench.json > report.yml - tags: - - process_result diff --git a/.github/workflows/benchmark/ansible/requirements.yml b/.github/workflows/benchmark/ansible/requirements.yml deleted file mode 100644 index 0718aca618..0000000000 --- a/.github/workflows/benchmark/ansible/requirements.yml +++ /dev/null @@ -1,6 +0,0 @@ -- src: https://github.com/keroro520/ansible-ckb - scm: git - version: main -- src: https://github.com/keroro520/ansible-ckb-benchmark - scm: git - version: main diff --git a/.github/workflows/benchmark/ansible/vars/all.yml b/.github/workflows/benchmark/ansible/vars/all.yml deleted file mode 100644 index 7d695ea4ef..0000000000 --- a/.github/workflows/benchmark/ansible/vars/all.yml +++ /dev/null @@ -1,34 +0,0 @@ -# TODO elegent way to assign bootnodes list - - -# ansible-ckb variables -ckb_workspace: "/var/lib/ckb" -ckb_data_dir: "{{ ckb_workspace }}/data" -secret_keys: - instance-0: "QmUpChEcQwBcnhTqoNXxynW2B4hNq9R2USgWZKHUnkWdGd" - instance-1: "QmcjX7xQxHJ2spbpb8uCD89LiCNbFsnFCLBA1Mu9SKgXjo" - instance-2: "Qmctiy6jRrpZnf7SCWjCskwetwKroW7bhZJc28PydsBevX" -ckb_network_secret_key: "{{ secret_keys[inventory_hostname] | default('') }}" -ckb_network_bootnodes: - - "/ip4/{{ hostvars['instance-0'].ansible_host }}/tcp/8115/p2p/{{ secret_keys['instance-0'] }}" -ckb_network_bootnode_mode: "true" -ckb_rpc_listen_address: "0.0.0.0:8114" -ckb_tx_pool_max_mem_size: "40_000_000" -ckb_tx_pool_max_cycles: "400_000_000_000" -ckb_tx_pool_min_fee_rate: "0" -ckb_chain_spec_file: "benchmark-spec.toml" - -# ansible-ckb miner variables -ckb_block_assembler: - key: "98400f6a67af07025f5959af35ed653d649f745b8f54bf3f07bef9bd605ee946" - code_hash: "0x9bd7e06f3ecf4be0f2fcd2188b23f1b9fcc88e5d4b65a8637b17723bbda3cce8" - args: "0x8883a512ee2383c01574a328f60eeccbb4d78240" - hash_type: "type" - message: "0x" -ckb_miner_workers: - - worker_type: "Dummy" - delay_type: "Constant" - value: 500 - -# ansible-ckb-benchmark variables -ckb_benchmark_owner_privkey: "{{ ckb_block_assembler.key }}" diff --git a/.github/workflows/benchmark/terraform/.gitignore b/.github/workflows/benchmark/terraform/.gitignore deleted file mode 100644 index dbf7ead575..0000000000 --- a/.github/workflows/benchmark/terraform/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -/.terraform -/terraform.tfstate -/terraform.tfstate.backup -/terraform.tfvars diff --git a/.github/workflows/benchmark/terraform/.terraform.lock.hcl b/.github/workflows/benchmark/terraform/.terraform.lock.hcl deleted file mode 100644 index 8334cdf136..0000000000 --- a/.github/workflows/benchmark/terraform/.terraform.lock.hcl +++ /dev/null @@ -1,42 +0,0 @@ -# This file is maintained automatically by "terraform init". -# Manual edits may be lost in future updates. - -provider "registry.terraform.io/hashicorp/aws" { - version = "2.70.0" - constraints = "~> 2.53" - hashes = [ - "h1:mM6eIaG1Gcrk47TveViXBO9YjY6nDaGukbED2bdo8Mk=", - "zh:01a5f351146434b418f9ff8d8cc956ddc801110f1cc8b139e01be2ff8c544605", - "zh:1ec08abbaf09e3e0547511d48f77a1e2c89face2d55886b23f643011c76cb247", - "zh:606d134fef7c1357c9d155aadbee6826bc22bc0115b6291d483bc1444291c3e1", - "zh:67e31a71a5ecbbc96a1a6708c9cc300bbfe921c322320cdbb95b9002026387e1", - "zh:75aa59ae6f0834ed7142c81569182a658e4c22724a34db5d10f7545857d8db0c", - "zh:76880f29fca7a0a3ff1caef31d245af2fb12a40709d67262e099bc22d039a51d", - "zh:aaeaf97ffc1f76714e68bc0242c7407484c783d604584c04ad0b267b6812b6dc", - "zh:ae1f88d19cc85b2e9b6ef71994134d55ef7830fd02f1f3c58c0b3f2b90e8b337", - "zh:b155bdda487461e7b3d6e3a8d5ce5c887a047e4d983512e81e2c8266009f2a1f", - "zh:ba394a7c391a26c4a91da63ad680e83bde0bc1ecc0a0856e26e9d62a4e77c408", - "zh:e243c9d91feb0979638f28eb26f89ebadc179c57a2bd299b5729fb52bd1902f2", - "zh:f6c05e20d9a3fba76ca5f47206dde35e5b43b6821c6cbf57186164ce27ba9f15", - ] -} - -provider "registry.terraform.io/hashicorp/null" { - version = "2.1.2" - constraints = "~> 2.1" - hashes = [ - "h1:l0/ASa/TB1exgqdi33sOkFakJL2zAQBu+q5LgO5/SxI=", - "zh:0cc7236c1fbf971b8bad1540a7d0c5ac4579248239fd1034c023b0b660a8d1d5", - "zh:16fc2d9b10cf9e5123bf956e7032c338cc93a03be1ca2e9f3d3b7014c0e866c7", - "zh:1e26465ff40ded59cef1a9e745752eef9744471e69094d12f8bc40a060e8cdb9", - "zh:3c7afd28076245f455d4348af6c124b73409722be4a73680d4e4709a4f25ea91", - "zh:4190a92567efaa444527f19b28d65fac1a01aeba907013b5dceacd9ba2a23709", - "zh:677963437316b088fc1aac70fe7d608d2917c46530c4a25ec86a43e9acaf2811", - "zh:69fe15f6b851ff82700bc749c50a9299770515617e677c18cba2cb697daaff36", - "zh:6b505cc60cc1494e1cab61590bca127b06dd894d0b2a7dcacd23862bce1f492b", - "zh:719aa11ad7be974085af595089478eb24d5a021bc7b08364fa6745d9eb473dac", - "zh:7375b02189e14efbfaab994cd05d81e3ff8e46041fae778598b3903114093a3e", - "zh:c406573b2084a08f8faa0451923fbeb1ca6c5a5598bf039589ec2db13aacc622", - "zh:fb11299a3b20711595713d126abbbe78c554eb5995b02db536e9253686798fb6", - ] -} diff --git a/.github/workflows/benchmark/terraform/README.md b/.github/workflows/benchmark/terraform/README.md deleted file mode 100644 index 7112dca3a7..0000000000 --- a/.github/workflows/benchmark/terraform/README.md +++ /dev/null @@ -1,24 +0,0 @@ -# Terraform Configuration Files Used For Benchmark Workflow - -These Terraform configuration files are part of the ["Benchmark" workflow](../../benchmark.yml). - -## AMI - -Read [`ami.tf`](./ami.tf) - -## Variables - -Read [`variables.tf`](./variables.tf) - -## Resources - -All resources are named with prefix [`var.prefix`](./variables.tf#L33) - -* 1 bastion node named `"bastion-0"` -* [`var.instances_count`](./variables.tf#L17) instances nodes named `"instance-*"` -* `aws_vpc` -* `aws_key_pair` - -## Outputs - -* [`ansible_hosts`](./main.tf#L161) diff --git a/.github/workflows/benchmark/terraform/ami.tf b/.github/workflows/benchmark/terraform/ami.tf deleted file mode 100644 index 74c5ce68c2..0000000000 --- a/.github/workflows/benchmark/terraform/ami.tf +++ /dev/null @@ -1,9 +0,0 @@ -data "aws_ami" "ubuntu" { - most_recent = true - owners = ["099720109477"] # Canonical - - filter { - name = "name" - values = ["ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-*"] - } -} diff --git a/.github/workflows/benchmark/terraform/main.tf b/.github/workflows/benchmark/terraform/main.tf deleted file mode 100644 index 233f4419d2..0000000000 --- a/.github/workflows/benchmark/terraform/main.tf +++ /dev/null @@ -1,184 +0,0 @@ -provider "aws" { - region = var.region - access_key = var.access_key - secret_key = var.secret_key -} - -resource "aws_vpc" "vpc" { - cidr_block = "10.0.0.0/16" - enable_dns_hostnames = true - tags = { - Name = "${var.prefix}-vpc" - } -} - -resource "aws_subnet" "subnet" { - vpc_id = aws_vpc.vpc.id - cidr_block = "${var.private_ip_prefix}.0/24" - map_public_ip_on_launch = true - tags = { - Name = "${var.prefix}-subnet" - } -} - -resource "aws_internet_gateway" "ig" { - vpc_id = aws_vpc.vpc.id - tags = { - Name = "${var.prefix}-ig" - } -} - -resource "aws_route" "internet_access" { - route_table_id = aws_vpc.vpc.main_route_table_id - destination_cidr_block = "0.0.0.0/0" - gateway_id = aws_internet_gateway.ig.id -} - -resource "aws_key_pair" "ssh" { - key_name = "${var.prefix}-ssh_key" - public_key = file(var.public_key_path) -} - -resource "aws_security_group" "sg_bastion" { - name = "${var.prefix}-sg_bastion" - vpc_id = aws_vpc.vpc.id - - ingress { - description = "ssh" - from_port = 22 - to_port = 22 - protocol = "tcp" - cidr_blocks = ["0.0.0.0/0"] - } - - ingress { - description = "all inbound from anywhere" - from_port = 0 - to_port = 0 - protocol = "-1" - cidr_blocks = ["0.0.0.0/0"] - } - - egress { - description = "all outbound to anywhere" - from_port = 0 - to_port = 0 - protocol = "-1" - cidr_blocks = ["0.0.0.0/0"] - } -} - -resource "aws_instance" "bastion" { - key_name = aws_key_pair.ssh.id - instance_type = var.instance_type_bastion - ami = data.aws_ami.ubuntu.id - vpc_security_group_ids = [aws_security_group.sg_bastion.id] - private_ip = var.private_ip_bastion - subnet_id = aws_subnet.subnet.id - - root_block_device { - volume_size = "60" - } - - connection { - type = "ssh" - host = aws_instance.bastion.public_ip - user = var.username - private_key = file(var.private_key_path) - } - - provisioner "remote-exec" { - inline = [ - "while [ ! -f /var/lib/cloud/instance/boot-finished ]; do echo 'Waiting for cloud-init...'; sleep 1; done", - ] - } - - tags = { - Name = "${var.prefix}-bastion" - } -} - -resource "aws_security_group" "sg_default" { - name = "${var.prefix}-sg-default" - description = "Allow inbound access from anywhere and outbound access to all" - vpc_id = aws_vpc.vpc.id - - ingress { - from_port = 0 - to_port = 0 - protocol = "-1" - cidr_blocks = ["0.0.0.0/0"] - } - - egress { - from_port = 0 - to_port = 0 - protocol = "-1" - cidr_blocks = ["0.0.0.0/0"] - } -} - -resource "aws_instance" "instance" { - count = var.instances_count - key_name = aws_key_pair.ssh.id - instance_type = var.instance_type - ami = data.aws_ami.ubuntu.id - vpc_security_group_ids = [aws_security_group.sg_default.id] - private_ip = "${var.private_ip_prefix}.${count.index + 100}" - subnet_id = aws_subnet.subnet.id - - root_block_device { - volume_size = "60" - } - - tags = { - Name = "${var.prefix}-instance-${count.index}" - Index = "${count.index}" - } -} - -resource "null_resource" "instance_provisioners" { - count = var.instances_count - - triggers = { - cluster_instance_ids = join(",", aws_instance.instance.*.id) - } - - connection { - bastion_host = aws_instance.bastion.public_ip - host = element(aws_instance.instance.*.private_ip, count.index) - user = var.username - private_key = file(var.private_key_path) - } - - provisioner "remote-exec" { - inline = [ - "while [ ! -f /var/lib/cloud/instance/boot-finished ]; do echo 'Waiting for cloud-init...'; sleep 1; done", - ] - } -} - -output "ansible_hosts" { - value = <> $GITHUB_ENV +function benchmark() { + $SCRIPT_PATH/script/benchmark.sh run + cp $JOB_DIRECTORY/ansible/ckb-bench.log $ANSIBLE_DIR + # TODO: copy report.yml to $ANSIBLE_DIR + $SCRIPT_PATH/script/benchmark.sh clean +} + +function github_report_error() { + $SCRIPT_PATH/script/ok.sh add_comment nervosnetwork/ckb 2372 "**Sync-Mainnet Report**:\nSync-Mainnet crashed" + + # double check + $SCRIPT_PATH/script/benchmark.sh clean +} + +function main() { + benchmark || github_report_error +} +main $* \ No newline at end of file diff --git a/network/src/protocols/identify/mod.rs b/network/src/protocols/identify/mod.rs index 46c6728796..2bf92453e7 100644 --- a/network/src/protocols/identify/mod.rs +++ b/network/src/protocols/identify/mod.rs @@ -3,7 +3,7 @@ use std::collections::HashMap; use std::sync::Arc; use std::time::{Duration, Instant}; -use ckb_logger::{debug, error, trace, warn}; +use ckb_logger::{debug, trace, warn}; use p2p::{ bytes::Bytes, context::{ProtocolContext, ProtocolContextMutRef, SessionContext}, @@ -90,7 +90,6 @@ pub trait Callback: Clone + Send { pub struct IdentifyProtocol { callback: T, remote_infos: HashMap, - secio_enabled: bool, global_ip_only: bool, } @@ -99,7 +98,6 @@ impl IdentifyProtocol { IdentifyProtocol { callback, remote_infos: HashMap::default(), - secio_enabled: true, global_ip_only: true, } } @@ -229,12 +227,6 @@ impl ServiceProtocol for IdentifyProtocol { fn connected(&mut self, context: ProtocolContextMutRef, version: &str) { let session = context.session; - if session.remote_pubkey.is_none() { - error!("IdentifyProtocol require secio enabled!"); - let _ = context.disconnect(session.id); - self.secio_enabled = false; - return; - } self.callback.register(&context, version); @@ -261,21 +253,15 @@ impl ServiceProtocol for IdentifyProtocol { } fn disconnected(&mut self, context: ProtocolContextMutRef) { - if self.secio_enabled { - let info = self - .remote_infos - .remove(&context.session.id) - .expect("RemoteInfo must exists"); - trace!("IdentifyProtocol disconnected from {:?}", info.peer_id); - self.callback.unregister(&context) - } + let info = self + .remote_infos + .remove(&context.session.id) + .expect("RemoteInfo must exists"); + trace!("IdentifyProtocol disconnected from {:?}", info.peer_id); + self.callback.unregister(&context) } fn received(&mut self, mut context: ProtocolContextMutRef, data: Bytes) { - if !self.secio_enabled { - return; - } - let session = context.session; match IdentifyMessage::decode(&data) { @@ -317,10 +303,6 @@ impl ServiceProtocol for IdentifyProtocol { } fn notify(&mut self, context: &mut ProtocolContext, _token: u64) { - if !self.secio_enabled { - return; - } - for (session_id, info) in &self.remote_infos { if !info.has_received && (info.connected_at + info.timeout) <= Instant::now() { debug!("{:?} receive identify message timeout", info.peer_id); diff --git a/network/src/protocols/ping.rs b/network/src/protocols/ping.rs index 9375df5efe..818c0a0ee1 100644 --- a/network/src/protocols/ping.rs +++ b/network/src/protocols/ping.rs @@ -160,33 +160,24 @@ impl ServiceProtocol for PingHandler { fn connected(&mut self, context: ProtocolContextMutRef, version: &str) { let session = context.session; - match session.remote_pubkey { - Some(_) => { - self.connected_session_ids - .entry(session.id) - .or_insert_with(|| PingStatus { - last_ping_sent_at: Instant::now(), - processing: false, - nonce: 0, - }); - debug!( - "proto id [{}] open on session [{}], address: [{}], type: [{:?}], version: {}", - context.proto_id, session.id, session.address, session.ty, version - ); - debug!("connected sessions are: {:?}", self.connected_session_ids); - // Register open ping protocol - self.network_state.with_peer_registry_mut(|reg| { - reg.get_peer_mut(session.id).map(|peer| { - peer.protocols.insert(context.proto_id, version.to_owned()); - }) - }); - } - None => { - if context.disconnect(session.id).is_err() { - debug!("disconnect fail"); - } - } - } + self.connected_session_ids + .entry(session.id) + .or_insert_with(|| PingStatus { + last_ping_sent_at: Instant::now(), + processing: false, + nonce: 0, + }); + debug!( + "proto id [{}] open on session [{}], address: [{}], type: [{:?}], version: {}", + context.proto_id, session.id, session.address, session.ty, version + ); + debug!("connected sessions are: {:?}", self.connected_session_ids); + // Register open ping protocol + self.network_state.with_peer_registry_mut(|reg| { + reg.get_peer_mut(session.id).map(|peer| { + peer.protocols.insert(context.proto_id, version.to_owned()); + }) + }); } fn disconnected(&mut self, context: ProtocolContextMutRef) { diff --git a/rpc/src/error.rs b/rpc/src/error.rs index db7a213369..e0ff9e17e7 100644 --- a/rpc/src/error.rs +++ b/rpc/src/error.rs @@ -171,15 +171,6 @@ impl RPCError { RPCError::custom_with_error(code, reject) } - /// Downcasts a CKB error if it is created from `Reject`. - pub fn downcast_submit_transaction_reject(err: &CKBError) -> Option<&Reject> { - use ckb_error::ErrorKind::SubmitTransaction; - match err.kind() { - SubmitTransaction => err.downcast_ref::(), - _ => None, - } - } - /// Creates an CKB error from `CKBError`. pub fn from_ckb_error(err: CKBError) -> Error { match err.kind() { diff --git a/rpc/src/module/pool.rs b/rpc/src/module/pool.rs index 72a07b7f3c..188f94e6b0 100644 --- a/rpc/src/module/pool.rs +++ b/rpc/src/module/pool.rs @@ -412,10 +412,7 @@ impl PoolRpc for PoolRpcImpl { let tx_hash = tx.hash(); match submit_tx.unwrap() { Ok(_) => Ok(tx_hash.unpack()), - Err(e) => match RPCError::downcast_submit_transaction_reject(&e) { - Some(reject) => Err(RPCError::from_submit_transaction_reject(reject)), - None => Err(RPCError::from_ckb_error(e)), - }, + Err(reject) => Err(RPCError::from_submit_transaction_reject(&reject)), } } diff --git a/rpc/src/tests/error.rs b/rpc/src/tests/error.rs index 2812763387..0d81c6a798 100644 --- a/rpc/src/tests/error.rs +++ b/rpc/src/tests/error.rs @@ -20,40 +20,34 @@ fn test_dao_error_from_ckb_error() { #[test] fn test_submit_transaction_error() { let min_fee_rate = FeeRate::from_u64(500); - let err: CKBError = Reject::LowFeeRate(min_fee_rate, 100, 50).into(); + let reject = Reject::LowFeeRate(min_fee_rate, 100, 50); assert_eq!( "PoolRejectedTransactionByMinFeeRate: The min fee rate is 500 shannons/KB, so the transaction fee should be 100 shannons at least, but only got 50", - RPCError::from_submit_transaction_reject(RPCError::downcast_submit_transaction_reject(&err).unwrap()).message + RPCError::from_submit_transaction_reject(&reject).message ); - let err: CKBError = Reject::ExceededMaximumAncestorsCount.into(); + let reject = Reject::ExceededMaximumAncestorsCount; assert_eq!( "PoolRejectedTransactionByMaxAncestorsCountLimit: Transaction exceeded maximum ancestors count limit, try send it later", - RPCError::from_submit_transaction_reject(RPCError::downcast_submit_transaction_reject(&err).unwrap()).message + RPCError::from_submit_transaction_reject(&reject).message ); - let err: CKBError = Reject::Full("size".to_owned(), 10).into(); + let reject = Reject::Full("size".to_owned(), 10); assert_eq!( "PoolIsFull: Transaction pool exceeded maximum size limit(10), try send it later", - RPCError::from_submit_transaction_reject( - RPCError::downcast_submit_transaction_reject(&err).unwrap() - ) - .message + RPCError::from_submit_transaction_reject(&reject).message ); - let err: CKBError = Reject::Duplicated(Byte32::new([0; 32])).into(); + let reject = Reject::Duplicated(Byte32::new([0; 32])); assert_eq!( "PoolRejectedDuplicatedTransaction: Transaction(Byte32(0x0000000000000000000000000000000000000000000000000000000000000000)) already exist in transaction_pool", - RPCError::from_submit_transaction_reject(RPCError::downcast_submit_transaction_reject(&err).unwrap()).message + RPCError::from_submit_transaction_reject(&reject).message ); - let err: CKBError = Reject::Malformed("cellbase like".to_owned()).into(); + let reject = Reject::Malformed("cellbase like".to_owned()); assert_eq!( "PoolRejectedMalformedTransaction: Malformed cellbase like transaction", - RPCError::from_submit_transaction_reject( - RPCError::downcast_submit_transaction_reject(&err).unwrap() - ) - .message + RPCError::from_submit_transaction_reject(&reject).message ); } diff --git a/script/Cargo.toml b/script/Cargo.toml index 73bce4cf28..8005efd1ed 100644 --- a/script/Cargo.toml +++ b/script/Cargo.toml @@ -21,8 +21,8 @@ ckb-traits = { path = "../traits", version = "= 0.102.0-pre" } byteorder = "1.3.1" ckb-types = {path = "../util/types", version = "= 0.102.0-pre"} ckb-hash = {path = "../util/hash", version = "= 0.102.0-pre"} -ckb-vm-definitions = "0.20.0-rc6" -ckb-vm = { version = "0.20.0-rc6", default-features = false } +ckb-vm-definitions = "0.20.0" +ckb-vm = { version = "0.20.0", default-features = false } faster-hex = "0.6" ckb-logger = { path = "../util/logger", version = "= 0.102.0-pre", optional = true } serde = { version = "1.0", features = ["derive"] } diff --git a/script/src/syscalls/load_cell_data.rs b/script/src/syscalls/load_cell_data.rs index 73746b4b65..0f33ceee70 100644 --- a/script/src/syscalls/load_cell_data.rs +++ b/script/src/syscalls/load_cell_data.rs @@ -9,7 +9,7 @@ use crate::{ use ckb_traits::CellDataProvider; use ckb_types::core::cell::CellMeta; use ckb_vm::{ - memory::{Memory, FLAG_EXECUTABLE, FLAG_FREEZED}, + memory::{Memory, FLAG_DIRTY, FLAG_EXECUTABLE, FLAG_FREEZED}, registers::{A0, A1, A2, A3, A4, A5, A7}, Error as VMError, Register, SupportMachine, Syscalls, }; @@ -107,7 +107,7 @@ impl<'a, DL: CellDataProvider + 'a> LoadCellData<'a, DL> { machine.memory_mut().init_pages( addr, memory_size, - FLAG_EXECUTABLE | FLAG_FREEZED, + FLAG_DIRTY | FLAG_EXECUTABLE | FLAG_FREEZED, Some(data), 0, )?; diff --git a/sync/src/relayer/transaction_hashes_process.rs b/sync/src/relayer/transaction_hashes_process.rs index b51b708163..e7baa7d2d5 100644 --- a/sync/src/relayer/transaction_hashes_process.rs +++ b/sync/src/relayer/transaction_hashes_process.rs @@ -45,8 +45,6 @@ impl<'a> TransactionHashesProcess<'a> { .collect() }; - state.add_ask_for_txs(self.peer, tx_hashes); - - Status::ok() + state.add_ask_for_txs(self.peer, tx_hashes) } } diff --git a/sync/src/status.rs b/sync/src/status.rs index 7ed7290b68..1c4fb6f93e 100644 --- a/sync/src/status.rs +++ b/sync/src/status.rs @@ -101,6 +101,8 @@ pub enum StatusCode { GetHeadersMissCommonAncestors = 414, /// Headers verified failed HeadersIsInvalid = 415, + /// Too many unknown transactions + TooManyUnknownTransactions = 416, /////////////////////////////////// // Warning 5xx // diff --git a/sync/src/types/mod.rs b/sync/src/types/mod.rs index 54fba7cae6..8775b119b8 100644 --- a/sync/src/types/mod.rs +++ b/sync/src/types/mod.rs @@ -1,7 +1,7 @@ use crate::block_status::BlockStatus; use crate::orphan_block_pool::OrphanBlockPool; use crate::utils::is_internal_db_error; -use crate::{FAST_INDEX, LOW_INDEX, NORMAL_INDEX, TIME_TRACE_SIZE}; +use crate::{Status, StatusCode, FAST_INDEX, LOW_INDEX, NORMAL_INDEX, TIME_TRACE_SIZE}; use ckb_app_config::SyncConfig; use ckb_chain::chain::ChainController; use ckb_chain_spec::consensus::Consensus; @@ -10,7 +10,8 @@ use ckb_constant::sync::{ BLOCK_DOWNLOAD_TIMEOUT, HEADERS_DOWNLOAD_HEADERS_PER_SECOND, HEADERS_DOWNLOAD_INSPECT_WINDOW, HEADERS_DOWNLOAD_TOLERABLE_BIAS_FOR_SINGLE_SAMPLE, INIT_BLOCKS_IN_TRANSIT_PER_PEER, MAX_BLOCKS_IN_TRANSIT_PER_PEER, MAX_HEADERS_LEN, MAX_OUTBOUND_PEERS_TO_PROTECT_FROM_DISCONNECT, - POW_INTERVAL, RETRY_ASK_TX_TIMEOUT_INCREASE, SUSPEND_SYNC_TIME, + MAX_UNKNOWN_TX_HASHES_SIZE, MAX_UNKNOWN_TX_HASHES_SIZE_PER_PEER, POW_INTERVAL, + RETRY_ASK_TX_TIMEOUT_INCREASE, SUSPEND_SYNC_TIME, }; use ckb_error::Error as CKBError; use ckb_logger::{debug, error, trace}; @@ -47,7 +48,6 @@ use ckb_types::core::EpochNumber; pub use header_map::HeaderMapLru as HeaderMap; const FILTER_SIZE: usize = 20000; -const MAX_UNKNOWN_TX_HASHES_SIZE: usize = 50000; const GET_HEADERS_CACHE_SIZE: usize = 10000; // TODO: Need discussed const GET_HEADERS_TIMEOUT: Duration = Duration::from_secs(15); @@ -1717,15 +1717,12 @@ impl SyncState { result } - pub fn add_ask_for_txs(&self, peer_index: PeerIndex, tx_hashes: Vec) { + pub fn add_ask_for_txs(&self, peer_index: PeerIndex, tx_hashes: Vec) -> Status { let mut unknown_tx_hashes = self.unknown_tx_hashes.lock(); - if unknown_tx_hashes.len() >= MAX_UNKNOWN_TX_HASHES_SIZE { - return; - } for tx_hash in tx_hashes .into_iter() - .take(MAX_UNKNOWN_TX_HASHES_SIZE - unknown_tx_hashes.len()) + .take(MAX_UNKNOWN_TX_HASHES_SIZE_PER_PEER) { match unknown_tx_hashes.entry(tx_hash) { keyed_priority_queue::Entry::Occupied(entry) => { @@ -1742,6 +1739,33 @@ impl SyncState { } } } + + // Check `unknown_tx_hashes`'s length after inserting the arrival `tx_hashes` + if unknown_tx_hashes.len() >= MAX_UNKNOWN_TX_HASHES_SIZE + || unknown_tx_hashes.len() + >= self.peers.state.len() * MAX_UNKNOWN_TX_HASHES_SIZE_PER_PEER + { + ckb_logger::warn!( + "unknown_tx_hashes is too long, len: {}", + unknown_tx_hashes.len() + ); + + let mut peer_unknown_counter = 0; + for (_hash, priority) in unknown_tx_hashes.iter() { + for peer in priority.peers.iter() { + if *peer == peer_index { + peer_unknown_counter += 1; + } + } + } + if peer_unknown_counter >= MAX_UNKNOWN_TX_HASHES_SIZE_PER_PEER { + return StatusCode::TooManyUnknownTransactions.into(); + } + + return Status::ignored(); + } + + Status::ok() } pub fn already_known_tx(&self, hash: &Byte32) -> bool { diff --git a/test/src/main.rs b/test/src/main.rs index cdf18c1991..1e86ed88a3 100644 --- a/test/src/main.rs +++ b/test/src/main.rs @@ -402,6 +402,7 @@ fn all_specs() -> Vec> { Box::new(PoolPersisted), Box::new(TransactionRelayBasic), Box::new(TransactionRelayLowFeeRate), + Box::new(TooManyUnknownTransactions), // TODO failed on poor CI server // Box::new(TransactionRelayMultiple), Box::new(RelayInvalidTransaction), diff --git a/test/src/specs/relay/mod.rs b/test/src/specs/relay/mod.rs index 9633a8cf13..dc909f9cda 100644 --- a/test/src/specs/relay/mod.rs +++ b/test/src/specs/relay/mod.rs @@ -2,6 +2,7 @@ mod block_relay; mod compact_block; mod get_block_proposal_process; mod get_block_transactions_process; +mod too_many_unknown_transactions; mod transaction_relay; mod transaction_relay_low_fee_rate; @@ -9,5 +10,6 @@ pub use block_relay::*; pub use compact_block::*; pub use get_block_proposal_process::ProposalRespondSizelimit; pub use get_block_transactions_process::*; +pub use too_many_unknown_transactions::TooManyUnknownTransactions; pub use transaction_relay::*; pub use transaction_relay_low_fee_rate::TransactionRelayLowFeeRate; diff --git a/test/src/specs/relay/too_many_unknown_transactions.rs b/test/src/specs/relay/too_many_unknown_transactions.rs new file mode 100644 index 0000000000..e5255f9226 --- /dev/null +++ b/test/src/specs/relay/too_many_unknown_transactions.rs @@ -0,0 +1,47 @@ +use crate::util::cell::gen_spendable; +use crate::util::transaction::always_success_transaction; +use crate::utils::{build_relay_tx_hashes, since_from_absolute_timestamp, wait_until}; +use crate::{Net, Node, Spec}; +use ckb_constant::sync::{MAX_RELAY_TXS_NUM_PER_BATCH, MAX_UNKNOWN_TX_HASHES_SIZE_PER_PEER}; +use ckb_network::SupportProtocols; +use ckb_types::packed::CellInput; + +pub struct TooManyUnknownTransactions; + +impl Spec for TooManyUnknownTransactions { + fn run(&self, nodes: &mut Vec) { + let node0 = &nodes[0]; + let mut net = Net::new( + self.name(), + node0.consensus(), + vec![SupportProtocols::Sync, SupportProtocols::Relay], + ); + net.connect(node0); + + // Send `MAX_UNKNOWN_TX_HASHES_SIZE_PER_PEER` transactions with a same input + let input = gen_spendable(node0, 1)[0].to_owned(); + let tx_template = always_success_transaction(node0, &input); + let txs = { + (0..MAX_UNKNOWN_TX_HASHES_SIZE_PER_PEER).map(|i| { + let since = since_from_absolute_timestamp(i as u64); + tx_template + .as_advanced_builder() + .set_inputs(vec![CellInput::new(input.out_point.clone(), since)]) + .build() + }) + }; + let tx_hashes = txs.map(|tx| tx.hash()).collect::>(); + assert!(MAX_RELAY_TXS_NUM_PER_BATCH >= tx_hashes.len()); + net.send( + node0, + SupportProtocols::Relay, + build_relay_tx_hashes(&tx_hashes), + ); + + let banned = wait_until(60, || node0.rpc_client().get_banned_addresses().len() == 1); + assert!( + banned, + "NetController should be banned cause TooManyUnknownTransactions" + ); + } +} diff --git a/tx-pool/src/service.rs b/tx-pool/src/service.rs index 8ef846d3eb..c95be9c530 100644 --- a/tx-pool/src/service.rs +++ b/tx-pool/src/service.rs @@ -11,7 +11,7 @@ use ckb_app_config::{BlockAssemblerConfig, TxPoolConfig}; use ckb_async_runtime::Handle; use ckb_chain_spec::consensus::Consensus; use ckb_channel::oneshot; -use ckb_error::{AnyError, Error}; +use ckb_error::AnyError; use ckb_jsonrpc_types::{BlockTemplate, TransactionWithStatus, TxStatus}; use ckb_logger::error; use ckb_logger::info; @@ -20,7 +20,7 @@ use ckb_snapshot::Snapshot; use ckb_stop_handler::{SignalSender, StopHandler, WATCH_INIT}; use ckb_types::{ core::{ - tx_pool::{TxPoolEntryInfo, TxPoolIds}, + tx_pool::{Reject, TxPoolEntryInfo, TxPoolIds}, BlockView, Cycle, TransactionView, UncleBlockView, Version, }, packed::{Byte32, ProposalShortId}, @@ -71,7 +71,7 @@ type BlockTemplateArgs = ( Option, ); -pub(crate) type SubmitTxResult = Result; +pub(crate) type SubmitTxResult = Result; type FetchTxRPCResult = Option<(bool, TransactionView)>; @@ -758,7 +758,7 @@ async fn process(mut service: TxPoolService, message: Message) { arguments: tx, }) => { let result = service.process_tx(tx, None).await; - if let Err(e) = responder.send(result.map_err(Into::into)) { + if let Err(e) = responder.send(result) { error!("responder send submit_tx result failed {:?}", e); }; } diff --git a/util/constant/src/sync.rs b/util/constant/src/sync.rs index 03703c20f4..852565c6bc 100644 --- a/util/constant/src/sync.rs +++ b/util/constant/src/sync.rs @@ -63,3 +63,10 @@ pub const BAD_MESSAGE_BAN_TIME: Duration = Duration::from_secs(5 * 60); /// Default ban time for sync useless // 10 minutes, peer have no common ancestor block pub const SYNC_USELESS_BAN_TIME: Duration = Duration::from_secs(10 * 60); + +/// The maximum number transaction hashes inside a `RelayTransactionHashes` message +pub const MAX_RELAY_TXS_NUM_PER_BATCH: usize = 32767; +/// The soft limit to the number of unknown transactions +pub const MAX_UNKNOWN_TX_HASHES_SIZE: usize = 50000; +/// The soft limit to the number of unknown transactions per peer +pub const MAX_UNKNOWN_TX_HASHES_SIZE_PER_PEER: usize = MAX_RELAY_TXS_NUM_PER_BATCH;