diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e5eba33e55428..f94f42fbbd0eb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -29,10 +29,16 @@ stages: - kubernetes - flaming-fir +image: ${REGISTRY}/substrate-ci-linux:latest +# image: parity/rust-builder:b2c2c71d-20200217 + variables: GIT_STRATEGY: fetch GIT_DEPTH: 100 - SCCACHE_DIR: "/ci-cache/${CI_PROJECT_NAME}/sccache" + CARGO_HOME: "/ci-cache/${CI_PROJECT_NAME}/cargo/${CI_COMMIT_REF_NAME}/${CI_JOB_NAME}" + CARGO_TARGET_DIR: "/ci-cache/${CI_PROJECT_NAME}/targets/${CI_COMMIT_REF_NAME}/${CI_JOB_NAME}" + REGISTRY: registry.parity.io/parity/infrastructure/scripts + # SCCACHE_DIR: "/ci-cache/${CI_PROJECT_NAME}/sccache" CARGO_INCREMENTAL: 0 CI_SERVER_NAME: "GitLab CI" DOCKER_OS: "debian:stretch" @@ -53,18 +59,33 @@ variables: paths: - artifacts/ -.kubernetes-build: &kubernetes-build - tags: - - kubernetes-parity-build - environment: - name: parity-build .docker-env: &docker-env - image: parity/rust-builder:latest before_script: - rustup show - cargo --version + # - mkdir -p ${CARGO_HOME}; touch ${CARGO_HOME}/config + # if there is no directory for this $CI_COMMIT_REF_NAME/$CI_JOB_NAME + # create such directory and + # create hardlinks recursively of all the files from the master/$CI_JOB_NAME if it exists + # - if [[ ! -d $CARGO_TARGET_DIR ]]; then + # mkdir -p /ci-cache/${CI_PROJECT_NAME}/targets/${CI_COMMIT_REF_NAME}; + # cp -al /ci-cache/${CI_PROJECT_NAME}/targets/master/${CI_JOB_NAME} + # /ci-cache/${CI_PROJECT_NAME}/targets/${CI_COMMIT_REF_NAME} || + # echo "_____No such target dir, proceeding from scratch_____"; + # fi - sccache -s + after_script: + - echo "_____ Clean target dir _____" + - cd ./target/release/ + - ls -a + - for binary in $(find . -maxdepth 1 -type f ! -size 0 -exec grep -IL . "{}" \; | cut -c 3- ) + do + rm -rf $binary + done + - find ./target/{debug,release} -maxdepth 1 -type f -delete; + - rm -f ./target/.rustc_info.json; + - rm -rf ./target/{debug,release}/{deps,.fingerprint}/ only: - master - /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 @@ -80,7 +101,7 @@ variables: dependencies: [] interruptible: true tags: - - linux-docker + - k2 .build-only: &build-only only: @@ -93,53 +114,6 @@ variables: #### stage: test -check-runtime: - stage: test - image: parity/tools:latest - <<: *kubernetes-build - only: - - /^[0-9]+$/ - variables: - GITLAB_API: "https://gitlab.parity.io/api/v4" - GITHUB_API_PROJECT: "parity%2Finfrastructure%2Fgithub-api" - script: - - ./.maintain/gitlab/check_runtime.sh - interruptible: true - allow_failure: true - -check-signed-tag: - stage: test - image: parity/tools:latest - <<: *kubernetes-build - only: - - tags - - /^v[0-9]+\.[0-9]+\.[0-9]+.*$/ - script: - - ./.maintain/gitlab/check_signed.sh - allow_failure: false - -check-line-width: - stage: test - image: parity/tools:latest - <<: *kubernetes-build - only: - - /^[0-9]+$/ - script: - - ./.maintain/gitlab/check_line_width.sh - interruptible: true - allow_failure: true - - - - -check-polkadot-companion-build: - stage: build - <<: *docker-env - script: - - ./.maintain/gitlab/check_polkadot_companion_build.sh - interruptible: true - allow_failure: true - cargo-audit: stage: test @@ -150,20 +124,6 @@ cargo-audit: - cargo audit allow_failure: true -cargo-deny: - stage: test - <<: *docker-env - script: - - cargo deny check --hide-inclusion-graph -c .maintain/deny.toml - after_script: - - echo "___The complete log is in the artifacts___" - - cargo deny check -c .maintain/deny.toml 2> deny.log - artifacts: - name: $CI_COMMIT_SHORT_SHA - expire_in: 3 days - when: always - paths: - - deny.log cargo-check-benches: stage: test @@ -183,14 +143,23 @@ cargo-check-subkey: - BUILD_DUMMY_WASM_BINARY=1 time cargo check --release - sccache -s +#### stage: build test-linux-stable: &test-linux - stage: test + stage: build <<: *docker-env + # debug variables: + # TODO: $ echo $CARGO_TARGET_DIR + # /ci-cache/substrate/targets/4460/build-linux-substrate + # https://gitlab.parity.io/parity/substrate/-/jobs/339591 + # TODO: doesn't reuse build-linux-substrate cache, needs checking locally + # CARGO_TARGET_DIR: "/ci-cache/${CI_PROJECT_NAME}/targets/${CI_COMMIT_REF_NAME}/build-linux-substrate" # Enable debug assertions since we are running optimized builds for testing # but still want to have debug assertions. RUSTFLAGS: -Cdebug-assertions=y + # needs: + # - build-linux-substrate except: variables: - $DEPLOY_TAG @@ -198,178 +167,15 @@ test-linux-stable: &test-linux - WASM_BUILD_NO_COLOR=1 time cargo test --all --release --verbose --locked |& tee output.log - sccache -s - - echo "____Test job successful, checking for warnings____" - - awk '/^warning:/,/^$/ { print }' output.log > ${CI_COMMIT_SHORT_SHA}_warnings.log - - if [ -s ${CI_COMMIT_SHORT_SHA}_warnings.log ]; then - cat ${CI_COMMIT_SHORT_SHA}_warnings.log; - exit 1; - else - echo "___No warnings___"; - fi - -test-dependency-rules: - stage: test - <<: *docker-env - except: - variables: - - $DEPLOY_TAG - script: - - .maintain/ensure-deps.sh -unleash-check: - stage: test - <<: *docker-env - only: - - master - - tags - script: - - cargo install cargo-unleash ${CARGO_UNLEASH_INSTALL_PARAMS} - - cargo unleash check ${CARGO_UNLEASH_PKG_DEF} -test-frame-staking: - stage: test - <<: *docker-env - variables: - # Enable debug assertions since we are running optimized builds for testing - # but still want to have debug assertions. - RUSTFLAGS: -Cdebug-assertions=y - RUST_BACKTRACE: 1 - except: - variables: - - $DEPLOY_TAG - script: - - cd frame/staking/ - - WASM_BUILD_NO_COLOR=1 time cargo test --release --verbose --no-default-features --features std - - sccache -s - -test-frame-examples-compile-to-wasm: - stage: test - <<: *docker-env - variables: - # Enable debug assertions since we are running optimized builds for testing - # but still want to have debug assertions. - RUSTFLAGS: -Cdebug-assertions=y - RUST_BACKTRACE: 1 - except: - variables: - - $DEPLOY_TAG - script: - - cd frame/example-offchain-worker/ - - cargo +nightly build --target=wasm32-unknown-unknown --no-default-features - - cd ../example - - cargo +nightly build --target=wasm32-unknown-unknown --no-default-features - - sccache -s - -test-wasmtime: - stage: test - <<: *docker-env - variables: - # Enable debug assertions since we are running optimized builds for testing - # but still want to have debug assertions. - RUSTFLAGS: -Cdebug-assertions=y - RUST_BACKTRACE: 1 - except: - variables: - - $DEPLOY_TAG - script: - - cd client/executor - - WASM_BUILD_NO_COLOR=1 time cargo test --release --verbose --features wasmtime - - sccache -s - -test-runtime-benchmarks: - stage: test - <<: *docker-env - variables: - # Enable debug assertions since we are running optimized builds for testing - # but still want to have debug assertions. - RUSTFLAGS: -Cdebug-assertions=y - RUST_BACKTRACE: 1 - except: - variables: - - $DEPLOY_TAG - script: - - cd bin/node/cli - - WASM_BUILD_NO_COLOR=1 time cargo test --release --verbose --features runtime-benchmarks - - sccache -s - -test-linux-stable-int: - <<: *test-linux - except: - refs: - - /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 - variables: - - $DEPLOY_TAG - script: - - echo "___Logs will be partly shown at the end in case of failure.___" - - echo "___Full log will be saved to the job artifacts only in case of failure.___" - - WASM_BUILD_NO_COLOR=1 RUST_LOG=sync=trace,consensus=trace,client=trace,state-db=trace,db=trace,forks=trace,state_db=trace,storage_cache=trace - time cargo test -p node-cli --release --verbose --locked -- --ignored - &> ${CI_COMMIT_SHORT_SHA}_int_failure.log - - sccache -s - after_script: - - awk '/FAILED|^error\[/,0' ${CI_COMMIT_SHORT_SHA}_int_failure.log - artifacts: - name: $CI_COMMIT_SHORT_SHA - when: on_failure - expire_in: 3 days - paths: - - ${CI_COMMIT_SHORT_SHA}_int_failure.log - -check-web-wasm: - stage: test - <<: *docker-env - except: - - /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 - script: - # WASM support is in progress. As more and more crates support WASM, we - # should add entries here. See https://github.com/paritytech/substrate/issues/2416 - - time cargo build --target=wasm32-unknown-unknown -p sp-io - - time cargo build --target=wasm32-unknown-unknown -p sp-runtime - - time cargo build --target=wasm32-unknown-unknown -p sp-std - - time cargo build --target=wasm32-unknown-unknown -p sc-client - - time cargo build --target=wasm32-unknown-unknown -p sc-consensus-aura - - time cargo build --target=wasm32-unknown-unknown -p sc-consensus-babe - - time cargo build --target=wasm32-unknown-unknown -p sp-consensus - - time cargo build --target=wasm32-unknown-unknown -p sc-telemetry - # Note: the command below is a bit weird because several Cargo issues prevent us from compiling the node in a more straight-forward way. - - time cargo +nightly build --manifest-path=bin/node/cli/Cargo.toml --no-default-features --features "browser" --target=wasm32-unknown-unknown -Z features=itarget - - sccache -s - -test-full-crypto-feature: - stage: test - <<: *docker-env - variables: - # Enable debug assertions since we are running optimized builds for testing - # but still want to have debug assertions. - RUSTFLAGS: -Cdebug-assertions=y - RUST_BACKTRACE: 1 - except: - variables: - - $DEPLOY_TAG - script: - - cd primitives/core/ - - time cargo +nightly build --verbose --no-default-features --features full_crypto - - cd ../application-crypto - - time cargo +nightly build --verbose --no-default-features --features full_crypto - - sccache -s - - -#### stage: build - -test-browser-node: - stage: build - <<: *docker-env - needs: - - job: check-web-wasm - artifacts: false - script: - - cargo +nightly test --target wasm32-unknown-unknown -p node-browser-testing -Z features=itarget +#### stage: publish build-linux-substrate: &build-binary - stage: build - <<: *collect-artifacts + stage: publish + # <<: *collect-artifacts <<: *docker-env - <<: *build-only + # <<: *build-only before_script: - mkdir -p ./artifacts/substrate/ except: @@ -377,20 +183,6 @@ build-linux-substrate: &build-binary - $DEPLOY_TAG script: - WASM_BUILD_NO_COLOR=1 time cargo build --release --verbose - - mv ./target/release/substrate ./artifacts/substrate/. - - echo -n "Substrate version = " - - if [ "${CI_COMMIT_TAG}" ]; then - echo "${CI_COMMIT_TAG}" | tee ./artifacts/substrate/VERSION; - else - ./artifacts/substrate/substrate --version | - sed -n -r 's/^substrate ([0-9.]+.*-[0-9a-f]{7,13})-.*$/\1/p' | - tee ./artifacts/substrate/VERSION; - fi - - sha256sum ./artifacts/substrate/substrate | tee ./artifacts/substrate/substrate.sha256 - - printf '\n# building node-template\n\n' - - ./.maintain/node-template-release.sh ./artifacts/substrate/substrate-node-template.tar.gz - - cp -r .maintain/docker/substrate.Dockerfile ./artifacts/substrate/ - - sccache -s build-linux-subkey: <<: *build-binary @@ -399,282 +191,3 @@ build-linux-subkey: script: - cd ./bin/utils/subkey - BUILD_DUMMY_WASM_BINARY=1 time cargo build --release --verbose - - cd - - - mv ./target/release/subkey ./artifacts/subkey/. - - echo -n "Subkey version = " - - ./artifacts/subkey/subkey --version | - sed -n -r 's/^subkey ([0-9.]+.*)/\1/p' | - tee ./artifacts/subkey/VERSION; - - sha256sum ./artifacts/subkey/subkey | tee ./artifacts/subkey/subkey.sha256 - - cp -r .maintain/docker/subkey.Dockerfile ./artifacts/subkey/ - - sccache -s - -build-rust-doc-release: - stage: build - <<: *docker-env - allow_failure: true - artifacts: - name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}-doc" - when: on_success - expire_in: 7 days - paths: - - ./crate-docs - <<: *build-only - script: - - rm -f ./crate-docs/index.html # use it as an indicator if the job succeeds - - BUILD_DUMMY_WASM_BINARY=1 RUSTDOCFLAGS="--html-in-header $(pwd)/.maintain/rustdoc-header.html" time cargo +nightly doc --release --all --verbose - - cp -R ./target/doc ./crate-docs - - echo "" > ./crate-docs/index.html - - sccache -s - -check-polkadot-companion-status: - stage: post-build-test - image: parity/tools:latest - <<: *kubernetes-build - only: - - /^[0-9]+$/ - script: - - ./.maintain/gitlab/check_polkadot_companion_status.sh - - - -trigger-contracts-ci: - stage: publish - needs: - - job: build-linux-substrate - artifacts: false - - job: test-linux-stable - artifacts: false - trigger: - project: parity/srml-contracts-waterfall - branch: master - strategy: depend - only: - - master - - schedules - -#### stage: publish - -.publish-docker-release: &publish-docker-release - <<: *build-only - <<: *kubernetes-build - image: docker:stable - services: - - docker:dind - before_script: - - test "$Docker_Hub_User_Parity" -a "$Docker_Hub_Pass_Parity" - || ( echo "no docker credentials provided"; exit 1 ) - - docker login -u "$Docker_Hub_User_Parity" -p "$Docker_Hub_Pass_Parity" - - docker info - script: - - cd ./artifacts/$PRODUCT/ - - VERSION="$(cat ./VERSION)" - - echo "${PRODUCT} version = ${VERSION}" - - test -z "${VERSION}" && exit 1 - - docker build - --build-arg VCS_REF="${CI_COMMIT_SHA}" - --build-arg BUILD_DATE="$(date -u '+%Y-%m-%dT%H:%M:%SZ')" - --tag $CONTAINER_IMAGE:$VERSION - --tag $CONTAINER_IMAGE:latest - --file $DOCKERFILE . - - docker push $CONTAINER_IMAGE:$VERSION - - docker push $CONTAINER_IMAGE:latest - -publish-docker-substrate: - stage: publish - <<: *publish-docker-release - # collect VERSION artifact here to pass it on to kubernetes - <<: *collect-artifacts - dependencies: - - build-linux-substrate - variables: - DOCKER_HOST: tcp://localhost:2375 - DOCKER_DRIVER: overlay2 - GIT_STRATEGY: none - PRODUCT: substrate - DOCKERFILE: $PRODUCT.Dockerfile - CONTAINER_IMAGE: parity/$PRODUCT - after_script: - - docker logout - # only VERSION information is needed for the deployment - - find ./artifacts/ -depth -not -name VERSION -type f -delete - -publish-docker-subkey: - stage: publish - <<: *publish-docker-release - dependencies: - - build-linux-subkey - variables: - DOCKER_HOST: tcp://localhost:2375 - DOCKER_DRIVER: overlay2 - GIT_STRATEGY: none - PRODUCT: subkey - DOCKERFILE: $PRODUCT.Dockerfile - CONTAINER_IMAGE: parity/$PRODUCT - after_script: - - docker logout - -publish-s3-release: - stage: publish - <<: *build-only - <<: *kubernetes-build - dependencies: - - build-linux-substrate - - build-linux-subkey - image: parity/awscli:latest - variables: - GIT_STRATEGY: none - BUCKET: "releases.parity.io" - PREFIX: "substrate/${ARCH}-${DOCKER_OS}" - script: - - aws s3 sync ./artifacts/ s3://${BUCKET}/${PREFIX}/$(cat ./artifacts/substrate/VERSION)/ - - echo "update objects in latest path" - - aws s3 sync s3://${BUCKET}/${PREFIX}/$(cat ./artifacts/substrate/VERSION)/ s3://${BUCKET}/${PREFIX}/latest/ - after_script: - - aws s3 ls s3://${BUCKET}/${PREFIX}/latest/ - --recursive --human-readable --summarize - - -publish-s3-doc: - stage: publish - image: parity/awscli:latest - allow_failure: true - dependencies: - - build-rust-doc-release - cache: {} - <<: *build-only - <<: *kubernetes-build - variables: - GIT_STRATEGY: none - BUCKET: "releases.parity.io" - PREFIX: "substrate-rustdoc" - script: - - test -r ./crate-docs/index.html || ( - echo "./crate-docs/index.html not present, build:rust:doc:release job not complete"; - exit 1 - ) - - aws s3 sync --delete --size-only --only-show-errors - ./crate-docs/ s3://${BUCKET}/${PREFIX}/ - after_script: - - aws s3 ls s3://${BUCKET}/${PREFIX}/ - --human-readable --summarize - -publish-draft-release: - stage: publish - image: parity/tools:latest - only: - - tags - - /^v[0-9]+\.[0-9]+\.[0-9]+.*$/ - script: - - ./.maintain/gitlab/publish_draft_release.sh - interruptible: true - allow_failure: true - -publish-to-crates-io: - stage: publish - <<: *docker-env - only: - - tags - - /^ci-release-.*$/ - script: - - cargo install cargo-unleash ${CARGO_UNLEASH_INSTALL_PARAMS} - - cargo unleash em-dragons --no-check ${CARGO_UNLEASH_PKG_DEF} - interruptible: true - allow_failure: true - -.deploy-template: &deploy - stage: kubernetes - when: manual - retry: 1 - image: parity/kubetools:latest - <<: *build-only - tags: - # this is the runner that is used to deploy it - - kubernetes-parity-build - before_script: - - test -z "${DEPLOY_TAG}" && - test -f ./artifacts/substrate/VERSION && - DEPLOY_TAG="$(cat ./artifacts/substrate/VERSION)" - - test "${DEPLOY_TAG}" || ( echo "Neither DEPLOY_TAG nor VERSION information available"; exit 1 ) - script: - - echo "Substrate version = ${DEPLOY_TAG}" - # or use helm to render the template - - helm template - --values ./.maintain/kubernetes/values.yaml - --set image.tag=${DEPLOY_TAG} - --set validator.keys=${VALIDATOR_KEYS} - ./.maintain/kubernetes | kubectl apply -f - --dry-run=false - - echo "# substrate namespace ${KUBE_NAMESPACE}" - - kubectl -n ${KUBE_NAMESPACE} get all - - echo "# substrate's nodes' external ip addresses:" - - kubectl get nodes -l node=substrate - -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{range @.status.addresses[?(@.type=="ExternalIP")]}{.address}{"\n"}{end}' - - echo "# substrate' nodes" - - kubectl -n ${KUBE_NAMESPACE} get pods - -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.spec.nodeName}{"\n"}{end}' - - echo "# wait for the rollout to complete" - - kubectl -n ${KUBE_NAMESPACE} rollout status statefulset/substrate - -# have environment:url eventually point to the logs - -.deploy-cibuild: &deploy-cibuild - <<: *deploy - dependencies: - - publish-docker-substrate - -.deploy-tag: &deploy-tag - <<: *deploy - only: - variables: - - $DEPLOY_TAG - -# have environment:url eventually point to the logs - -deploy-ew3: - <<: *deploy-cibuild - environment: - name: parity-prod-ew3 - -deploy-ue1: - <<: *deploy-cibuild - environment: - name: parity-prod-ue1 - -deploy-ew3-tag: - <<: *deploy-tag - environment: - name: parity-prod-ew3 - -deploy-ue1-tag: - <<: *deploy-tag - environment: - name: parity-prod-ue1 - -.validator-deploy: &validator-deploy - # script will fail if there is no artifacts/substrate/VERSION - <<: *build-only - stage: flaming-fir - dependencies: - - build-linux-substrate - image: parity/azure-ansible:v1 - allow_failure: true - when: manual - tags: - - linux-docker - -validator 1 4: - <<: *validator-deploy - script: - - ./.maintain/flamingfir-deploy.sh flamingfir-validator1 -validator 2 4: - <<: *validator-deploy - script: - - ./.maintain/flamingfir-deploy.sh flamingfir-validator2 -validator 3 4: - <<: *validator-deploy - script: - - ./.maintain/flamingfir-deploy.sh flamingfir-validator3 -validator 4 4: - <<: *validator-deploy - script: - - ./.maintain/flamingfir-deploy.sh flamingfir-validator4