bnb-chain · buddh0 · Apr 26, 2024
diff --git a/.env b/.env
@@ -5,14 +5,14 @@ INIT_HOLDER="0x04d63aBCd2b9b1baa327f2Dda0f873F197ccd186"
 # INIT_HOLDER_PRV="59ba8068eb256d520179e903f43dacf6d8d57d72bd306e1bd603fdb8c8da10e8"
 RPC_URL="http://127.0.0.1:8545"
 GENESIS_COMMIT="34618f607f8356cf147dde6a69fae150bd53d5bf" # fermi commit
-PASSED_FORK_DELAY=40
+PASSED_FORK_DELAY=600
 LAST_FORK_MORE_DELAY=10
 FullImmutabilityThreshold=2048
 MinBlocksForBlobRequests=576
 DefaultExtraReserveForBlobRequests=32
 BreatheBlockInterval=1200
 useLatestBscClient=false
-EnableSentryNode=false
-EnableFullNode=false
-RegisterNodeID=false
-EnableEVNWhitelist=false
+EnableSentryNode=true
+EnableFullNode=true
+RegisterNodeID=true
+EnableEVNWhitelist=true
diff --git a/bsc_cluster.sh b/bsc_cluster.sh
@@ -9,12 +9,15 @@ basedir=$(
 )
 workspace=${basedir}
 source ${workspace}/.env
-size=$((BSC_CLUSTER_SIZE))
+source ${workspace}/qa-env-resource/machines_meta.sh # including machine ips and ids, don't upload!!!
+validator_ips=(${validator_ips_comma//,/ })
+size=${#validator_ips[@]}
 stateScheme="hash"
 dbEngine="leveldb"
 gcmode="full"
 sleepBeforeStart=15
 sleepAfterStart=10
+copyDir="bsc-qa"
 
 # stop geth client
 function exit_previous() {
@@ -57,7 +60,7 @@ function reset_genesis() {
     mv genesis-template.json.bk genesis-template.json
     mv scripts/init_holders.template.bk scripts/init_holders.template
 
-    poetry install --no-root
+    # poetry install --no-root
     npm install
     rm -rf lib/forge-std
     forge install --no-git foundry-rs/forge-std@v1.7.3
@@ -83,28 +86,49 @@ function prepare_config() {
         mkdir -p ${targetDir} && cd ${targetDir}
         cp ${workspace}/keys/password.txt ./
         cp ${workspace}/.local/hardforkTime.txt ./
+        cp ${workspace}/qa-env-resource/* ./ && rm -f upgrade-single*
+        sed -i -e "s/{{validatorAddr}}/${cons_addr}/g"  chaind.sh && rm -f chaind.sh.bak
         bbcfee_addrs=${fee_addr}
         powers="0x000001d1a94a2000" #2000000000000
         mv ${workspace}/.local/bls${i}/bls ./ && rm -rf ${workspace}/.local/bls${i}
         vote_addr=0x$(cat ./bls/keystore/*json | jq .pubkey | sed 's/"//g')
         echo "${cons_addr},${bbcfee_addrs},${fee_addr},${powers},${vote_addr}" >> ${workspace}/genesis/validators.conf
         if [ ${EnableSentryNode} = true ]; then
-            mkdir -p ${workspace}/.local/sentry${i}
+            targetDir=${workspace}/.local/sentry${i}
+            mkdir -p ${targetDir} && cd ${targetDir}
+            cp ${workspace}/.local/hardforkTime.txt ./
+            cp ${workspace}/qa-env-resource/* ./ && rm -f upgrade-single*
+            sed -i -e '/--mine/d' chaind.sh
+            sed -i -e 's/workdir="validator"/workdir="sentry"/g' chaind.sh
+            sed -i -e 's/bin="bsc"/bin="sentry"/g' chaind.sh
+            sed -i -e "s/portInc=0/portInc=2/g" chaind.sh
+            rm -f chaind.sh.bak
+            sed -i -e 's/workdir="validator"/workdir="sentry"/g' init.sh
+            sed -i -e 's/bin="bsc"/bin="sentry"/g' init.sh
+            rm -f init.sh.bak
+            mv bsc.service sentry.service
+            sed -i -e 's/validator/sentry/g' sentry.service
+            sed -i -e 's/bsc/sentry/g' sentry.service
+            rm -f sentry.service.bak
         fi
     done
     if [ ${EnableFullNode} = true ]; then
-        mkdir -p ${workspace}/.local/fullnode0
+        targetDir=${workspace}/.local/fullnode0
+        mkdir -p ${targetDir} && cd ${targetDir}
+        cp ${workspace}/.local/hardforkTime.txt ./
+        cp ${workspace}/qa-env-resource/* ./ && rm -f upgrade-single*
+        sed -i -e '/--mine/d' chaind.sh && rm -f chaind.sh.bak
     fi
     rm -f ${workspace}/.local/hardforkTime.txt
 
     cd ${workspace}/genesis/
     git checkout HEAD contracts
     sed -i -e  's/alreadyInit = true;/turnLength = 16;alreadyInit = true;/' ${workspace}/genesis/contracts/BSCValidatorSet.sol
     sed -i -e  's/public onlyCoinbase onlyZeroGasPrice {/public onlyCoinbase onlyZeroGasPrice {if (block.number < 2000) return;/' ${workspace}/genesis/contracts/BSCValidatorSet.sol
-    
-    poetry run python -m scripts.generate generate-validators
-    poetry run python -m scripts.generate generate-init-holders "${initHolders}"
-    poetry run python -m scripts.generate dev \
+
+    python3 -m scripts.generate generate-validators
+    python3 -m scripts.generate generate-init-holders "${initHolders}"
+    python3 -m scripts.generate dev \
       --dev-chain-id "${CHAIN_ID}" \
       --init-burn-ratio "1000" \
       --init-felony-slash-scope "60" \
@@ -142,10 +166,10 @@ function initNetwork() {
 
     init_extra_args=""
     if [ ${EnableSentryNode} = true ]; then
-        init_extra_args="--init.sentrynode-size ${size} --init.sentrynode-ports 30411"
+        init_extra_args="--init.sentrynode-size ${size} --init.sentrynode-ips ${sentry_ips_comma}"
     fi
     if [ ${EnableFullNode} = true ]; then
-        init_extra_args="${init_extra_args} --init.fullnode-size 1 --init.fullnode-ports 30511"
+        init_extra_args="${init_extra_args} --init.fullnode-size 1 --init.fullnode-ips ${fullnode_ips_comma}"
     fi
     if [ "${RegisterNodeID}" = true ]; then
         if [ "${EnableSentryNode}" = true ]; then
@@ -161,7 +185,7 @@ function initNetwork() {
             init_extra_args="${init_extra_args} --init.evn-validator-whitelist"
         fi
     fi
-    ${workspace}/bin/geth init-network --init.dir ${workspace}/.local --init.size=${size} --config ${workspace}/config.toml ${init_extra_args} ${workspace}/genesis/genesis.json
+    ${workspace}/bin/geth init-network --init.dir ${workspace}/.local --init.size=${size} --init.ips "${validator_ips_comma}" --config ${workspace}/qa-env-resource/config.toml ${init_extra_args} ${workspace}/genesis/genesis.json
     rm -f ${workspace}/*bsc.log*
     for ((i = 0; i < size; i++)); do
         sed -i -e '/"<nil>"/d' ${workspace}/.local/node${i}/config.toml
@@ -175,14 +199,21 @@ function initNetwork() {
         rm -f ${workspace}/.local/node${i}/*bsc.log*
 
         if [ ${EnableSentryNode} = true ]; then
+            sed -i -e 's/:30311/:30611/g' ${workspace}/.local/node${i}/config.toml
+            sed -i -e 's/:30411/:30311/g' ${workspace}/.local/node${i}/config.toml
+
             sed -i -e '/"<nil>"/d' ${workspace}/.local/sentry${i}/config.toml
+            sed -i -e 's/:30311/:30611/g' ${workspace}/.local/sentry${i}/config.toml
+            sed -i -e 's/:30411/:30311/g' ${workspace}/.local/sentry${i}/config.toml
             initLog=${workspace}/.local/sentry${i}/init.log
             ${workspace}/bin/geth --datadir ${workspace}/.local/sentry${i} init --state.scheme path --db.engine pebble ${workspace}/genesis/genesis.json  > "${initLog}" 2>&1
             rm -f ${workspace}/.local/sentry${i}/*bsc.log*
         fi
     done
     if [ ${EnableFullNode} = true ]; then
         sed -i -e '/"<nil>"/d' ${workspace}/.local/fullnode0/config.toml
+        sed -i -e 's/:30411/:30311/g' ${workspace}/.local/fullnode0/config.toml
+        sed -i -e 's/:30511/:30311/g' ${workspace}/.local/fullnode0/config.toml
         sed -i -e 's/EnableEVNFeatures = true/EnableEVNFeatures = false/g' ${workspace}/.local/fullnode0/config.toml
         initLog=${workspace}/.local/fullnode0/init.log
         ${workspace}/bin/geth --datadir ${workspace}/.local/fullnode0 init --state.scheme path --db.engine pebble ${workspace}/genesis/genesis.json  > "${initLog}" 2>&1
@@ -272,14 +303,78 @@ function native_start() {
 }
 
 function register_stakehub(){
-    # wait feynman enable
-    sleep 45
     for ((i = 0; i < size; i++));do
         ${workspace}/create-validator/create-validator --consensus-key-dir ${workspace}/keys/validator${i} --vote-key-dir ${workspace}/keys/bls${i} \
             --password-path ${workspace}/keys/password.txt --amount 20001 --validator-desc Val${i} --rpc-url ${RPC_URL}
     done
 }
 
+function remote_reset_config() {
+    rm -rf /mnt/efs/${copyDir}/clusterNetwork
+    cp -r ${workspace}/.local /mnt/efs/${copyDir}/clusterNetwork
+    ips=(${validator_ips_comma//,/ })
+    for ((i=0;i<${#ips[@]};i++));do
+        dst_id=${ips2ids[${ips[i]}]}
+        if [ ${EnableSentryNode} = true ]; then
+            aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript"   --parameters commands="sudo \cp -f /mnt/efs/${copyDir}/clusterNetwork/sentry${i}/config.toml /server/sentry/"
+	    aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript"   --parameters commands="sudo \cp -f /mnt/efs/${copyDir}/clusterNetwork/sentry${i}/chaind.sh /server/sentry/"
+        fi
+        aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript"   --parameters commands="sudo \cp -f /mnt/efs/${copyDir}/clusterNetwork/node${i}/config.toml /server/validator/"
+	aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript"   --parameters commands="sudo \cp -f /mnt/efs/${copyDir}/clusterNetwork/node${i}/chaind.sh /server/validator/"
+    done
+     if [ ${EnableFullNode} = true ]; then
+        fullnode_ips=(${fullnode_ips_comma//,/ })
+        dst_id=${ips2ids[${fullnode_ips[0]}]}
+        aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript"   --parameters commands="sudo \cp -f /mnt/efs/${copyDir}/clusterNetwork/fullnode0/config.toml /server/validator/"
+	aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript"   --parameters commands="sudo \cp -f /mnt/efs/${copyDir}/clusterNetwork/fullnode0/chaind.sh /server/validator/"
+    fi
+}
+
+function remote_start() {
+    rm -rf /mnt/efs/${copyDir}/clusterNetwork
+    cp -r ${workspace}/.local /mnt/efs/${copyDir}/clusterNetwork
+    for dst_id in ${ips2ids[@]}; do
+        # Always stop sentry nodes, regardless of the target setup.
+        aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript"   --parameters commands="sudo service sentry stop"
+        aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript"   --parameters commands="sudo service bsc stop"
+    done
+    sleep 80
+    cp ${workspace}/bin/geth /mnt/efs/${copyDir}/clusterNetwork/
+    ips=(${validator_ips_comma//,/ })
+    for ((i=0;i<${#ips[@]};i++));do
+        dst_id=${ips2ids[${ips[i]}]}
+        aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript"   --parameters commands="sudo cp /mnt/efs/${copyDir}/clusterNetwork/geth /tmp/geth && sudo bash -x /mnt/efs/${copyDir}/clusterNetwork/node${i}/init.sh"
+    done
+    if [ ${EnableSentryNode} = true ]; then
+        sleep 20
+        for ((i=0;i<${#ips[@]};i++));do
+            dst_id=${ips2ids[${ips[i]}]}
+            aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript"   --parameters commands="sudo cp /mnt/efs/${copyDir}/clusterNetwork/geth /tmp/geth && sudo bash -x /mnt/efs/${copyDir}/clusterNetwork/sentry${i}/init.sh"
+        done
+    fi
+    if [ ${EnableFullNode} = true ]; then
+        fullnode_ips=(${fullnode_ips_comma//,/ })
+        dst_id=${ips2ids[${fullnode_ips[0]}]}
+        aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript"   --parameters commands="sudo cp /mnt/efs/${copyDir}/clusterNetwork/geth /tmp/geth && sudo bash -x /mnt/efs/${copyDir}/clusterNetwork/fullnode0/init.sh"
+    fi
+}
+
+function remote_upgrade() {
+    cp ${workspace}/bin/geth /mnt/efs/${copyDir}/clusterNetwork/
+    if [ ${EnableSentryNode} = true ]; then
+        cp ${workspace}/qa-env-resource/upgrade-single-sentry.sh /mnt/efs/${copyDir}/clusterNetwork/
+    fi
+    cp ${workspace}/qa-env-resource/upgrade-single-validator.sh /mnt/efs/${copyDir}/clusterNetwork/
+    for dst_id in ${ips2ids[@]}; do
+        if [ ${EnableSentryNode} = true ]; then
+            aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript" \
+            --parameters commands="sudo cp /mnt/efs/${copyDir}/clusterNetwork/geth /tmp/geth && sudo cp /mnt/efs/${copyDir}/clusterNetwork/upgrade-single-sentry.sh /tmp/ && sudo bash -x /tmp/upgrade-single-sentry.sh"
+        fi
+        aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript" \
+            --parameters commands="sudo cp /mnt/efs/${copyDir}/clusterNetwork/geth /tmp/geth && sudo cp /mnt/efs/${copyDir}/clusterNetwork/upgrade-single-validator.sh /tmp/ && sudo bash -x /tmp/upgrade-single-validator.sh"
+    done
+}
+
 CMD=$1
 ValidatorIdx=$2
 case ${CMD} in
@@ -290,7 +385,7 @@ reset)
     reset_genesis
     prepare_config
     initNetwork
-    native_start
+    native_start 
     register_stakehub
     ;;
 stop)
@@ -303,7 +398,30 @@ restart)
     exit_previous $ValidatorIdx
     native_start $ValidatorIdx
     ;;
+remote_reset)
+    create_validator
+    reset_genesis
+    prepare_config
+    initNetwork
+    remote_start
+    # to prevent stuck
+    sleep 50
+    register_stakehub
+    ;;
+remote_reset_config)
+    create_validator
+    reset_genesis
+    prepare_config
+    initNetwork
+    remote_reset_config
+    ;;
+remote_upgrade)
+    remote_upgrade
+    ;;
+register_stakehub)
+    register_stakehub
+    ;;
 *)
-    echo "Usage: bsc_cluster.sh | reset | stop [vidx]| start [vidx]| restart [vidx]"
+    echo "Usage: bsc_cluster.sh | reset | stop [vidx]| start [vidx]| restart [vidx]| remote_reset | remote_upgrade | register_stakehub"
     ;;
 esac
diff --git a/qa-env-resource/bsc.service b/qa-env-resource/bsc.service
@@ -0,0 +1,18 @@
+[Unit]
+Description=bsc
+After=network.target
+
+[Service]
+Type=simple
+User=root
+Group=root
+ExecStart=/server/validator/chaind.sh -start
+ExecReload=/server/validator/chaind.sh -restart
+ExecStop=/server/validator/chaind.sh -stop
+PrivateTmp=true
+LimitNOFILE=10000
+StartLimitInterval=0
+TimeoutStopSec=120
+
+[Install]
+WantedBy=multi-user.target
diff --git a/qa-env-resource/chaind.sh b/qa-env-resource/chaind.sh
@@ -0,0 +1,80 @@
+#!/bin/bash
+
+set -e
+
+export GOGC=200
+# default values
+workdir="validator"
+bin="bsc"
+portInc=0
+FullImmutabilityThreshold=90000
+MinBlocksForBlobRequests=524288
+DefaultExtraReserveForBlobRequests=28800
+BreatheBlockInterval=600
+LAST_FORK_MORE_DELAY=200
+
+function startChaind() {
+    workspace=/server/${workdir}
+
+    PassedForkTime=`cat ${workspace}/hardforkTime.txt|grep passedHardforkTime|awk -F" " '{print $NF}'`
+    LastHardforkTime=$(expr ${PassedForkTime} + ${LAST_FORK_MORE_DELAY})
+    initLog=${workspace}/init.log
+    rialtoHash=`cat ${initLog}|grep "database=chaindata"|awk -F"=" '{print $NF}'|awk -F'"' '{print $1}'`
+
+    ip=`ifconfig eth0|grep inet|grep -v inet6 |awk '{ print $2 }'`
+    sed -i -e "s?FileRoot = \"\"?FileRoot = \"/mnt/efs/${workdir}/${ip}/\"?g" /server/${workdir}/config.toml
+    mkdir -p /mnt/efs/${workdir}/${ip}
+    HTTPPort=$((8545 + ${portInc}))
+    WSPort=${HTTPPort}
+    MetricsPort=$((6060 + ${portInc}))
+    PProfPort=$((${MetricsPort} + 1))
+    ${workspace}/${bin} --config ${workspace}/config.toml \
+        --mine --vote --unlock {{validatorAddr}} --miner.etherbase {{validatorAddr}} --password ${workspace}/password.txt --blspassword ${workspace}/password.txt \
+        --datadir ${workspace} \
+        --rpc.allow-unprotected-txs --allow-insecure-unlock \
+        --ws --ws.addr ${ip} --ws.port ${WSPort} --http --http.addr ${ip} --http.port ${HTTPPort} --http.corsdomain "*" \
+        --metrics --metrics.addr 0.0.0.0 --metrics.port ${MetricsPort} \
+        --pprof --pprof.port ${PProfPort} \
+        --syncmode full --monitor.maliciousvote \
+        --cache 10480 \
+        --rialtohash ${rialtoHash} --override.passedforktime ${PassedForkTime} --override.lorentz ${PassedForkTime} --override.maxwell ${PassedForkTime} --override.fermi ${LastHardforkTime} \
+        --override.immutabilitythreshold ${FullImmutabilityThreshold} --override.breatheblockinterval ${BreatheBlockInterval} \
+        --override.minforblobrequest ${MinBlocksForBlobRequests} --override.defaultextrareserve ${DefaultExtraReserveForBlobRequests} \
+        >> /mnt/efs/${workdir}/${ip}/bscnode.log 2>&1
+}
+
+function stopChaind() {
+    pid=`ps -ef | grep /server/${workdir}/${bin} | grep -v grep | awk '{print $2}'`
+    if [ -n "$pid" ]; then
+        kill -TERM $pid
+        for((i=1;i<=40;i++));
+        do
+            pid=`ps -ef | grep /server/${workdir}/${bin} | grep -v grep | awk '{print $2}'`
+            if [ -z "$pid" ]; then
+                break
+            fi
+            sleep 10
+        done
+    fi
+}
+
+CMD=$1
+
+case $CMD in
+-start)
+    echo "start"
+    startChaind
+    ;;
+-stop)
+    echo "stop"
+    stopChaind
+    ;;
+-restart)
+    stopChaind
+    sleep 3
+    startChaind
+    ;;
+*)
+    echo "Usage: chaind.sh -start | -stop | -restart .Or use systemctl start | stop | restart ${bin}.service "
+    ;;
+esac