From dd6f5608c4e1ebaebceeb01d61288245b457477c Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Tue, 19 Mar 2019 22:03:15 +0000 Subject: [PATCH 01/16] enable self_test test Signed-off-by: Ashley Pittman --- test/common_methods.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/common_methods.py b/test/common_methods.py index a2612be3..a7b113e5 100644 --- a/test/common_methods.py +++ b/test/common_methods.py @@ -850,7 +850,8 @@ def test_self_test(self): self_test = find_executable('self_test') if not self_test: - cart_prefix = os.getenv("IOF_CART_PREFIX", None) + cart_prefix = os.getenv("IOF_CART_PREFIX", + iofcommontestsuite.CART_PREFIX) if not cart_prefix: self.skipTest('Could not find self_test binary') self_test = os.path.join(cart_prefix, 'bin', 'self_test') @@ -861,7 +862,7 @@ def test_self_test(self): environ['OFI_INTERFACE'] = self.ofi_interface cmd = [self_test, '--singleton', '--path', self.cnss_prefix, '--group-name', 'IONSS', '-e' '0:0', - '-r', '1000', '-s' '0 0,0 128,128 0'] + '-r', '50', '-s' '0 0,0 128,128 0'] log_top_dir = os.getenv("IOF_TESTLOG", os.path.join(os.path.dirname( @@ -879,7 +880,7 @@ def test_self_test(self): outfile.write("{!s}\n Command: {!s} \n{!s}\n".format( ("=" * 40), (" ".join(cmd)), ("=" * 40))) outfile.flush() - procrtn = subprocess.call(cmd, timeout=180, env=environ, + procrtn = subprocess.call(cmd, timeout=5 * 60, env=environ, stdout=outfile, stderr=errfile) except (FileNotFoundError) as e: self.logger.info("Testnss: %s", \ From b4ace71d17064b27b7a4aa64dc792852a864888d Mon Sep 17 00:00:00 2001 From: Parallels Date: Wed, 20 Mar 2019 10:29:12 +0000 Subject: [PATCH 02/16] Use a .log suffix for self_test log files so they're archived correctly. Have the contents of the output file printed by the test itself so it's easier to locate. Signed-off-by: Parallels --- test/common_methods.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/test/common_methods.py b/test/common_methods.py index a7b113e5..f074488f 100644 --- a/test/common_methods.py +++ b/test/common_methods.py @@ -871,8 +871,8 @@ def test_self_test(self): if not os.path.exists(log_path): os.makedirs(log_path) - cmdfileout = os.path.join(log_path, "self_test.out") - cmdfileerr = os.path.join(log_path, "self_test.err") + cmdfileout = os.path.join(log_path, "self_test.out.log") + cmdfileerr = os.path.join(log_path, "self_test.err.log") procrtn = -1 try: with open(cmdfileout, mode='w') as outfile, \ @@ -888,6 +888,12 @@ def test_self_test(self): except (IOError) as e: self.logger.info("Testnss: Error opening the log files: %s", \ e.errno) + + with open(cmdfileout, "r") as fd: + for line in fd.readlines(): + print(line.strip()) + fd.close() + if procrtn != 0: self.fail("cart self test failed: %s" % procrtn) From b9ce1bba524612a3b607285906bba74990926822 Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Wed, 20 Mar 2019 18:14:24 +0000 Subject: [PATCH 03/16] Log self-test debugging to file. Signed-off-by: Ashley Pittman --- test/common_methods.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/common_methods.py b/test/common_methods.py index f074488f..6d193b10 100644 --- a/test/common_methods.py +++ b/test/common_methods.py @@ -873,6 +873,7 @@ def test_self_test(self): os.makedirs(log_path) cmdfileout = os.path.join(log_path, "self_test.out.log") cmdfileerr = os.path.join(log_path, "self_test.err.log") + environ['D_LOG_FILE'] = os.path.join(log_path, 'self_test_cart.log') procrtn = -1 try: with open(cmdfileout, mode='w') as outfile, \ From b7bcba6a36c4989d88e5994c7a8e1e4857e0f52e Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Tue, 19 Mar 2019 08:41:38 -0400 Subject: [PATCH 04/16] Build and test IOF using componment RPMs... Instead of building components from source. Add component RPMs to CentOS 7 Dockerfile. Harmonize the Dockerfile with the cart Dockerfile to encourage high cache efficiency. Signed-off-by: Brian J. Murrell --- Jenkinsfile | 62 +++++++++++++++++++++++++++- utils/docker/Dockerfile.centos:7 | 70 +++++++++++++++++++++++++------- 2 files changed, 116 insertions(+), 16 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 2d4846db..660b2b12 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -39,6 +39,27 @@ def sanitized_JOB_NAME = JOB_NAME.toLowerCase().replaceAll('/', '-').replaceAll('%2f', '-') +long get_timestamp() { + Date date = new Date() + return date.getTime() +} + +String get_deps_build_vars() { + def deps = [ + 'MERCURY' : '1.0.1-2', + 'OMPI' : '3.0.0rc4-3', + 'PMIX' : '2.1.1-2', + 'LIBFABRIC': '1.7.1rc1-1' + ] + def buildargs = "" + deps.each { + dep, ver -> buildargs +="--build-arg ${dep}=" + + dep.toLowerCase() + "-${ver} " + } + + return buildargs +} + pipeline { agent { label 'lightweight' } @@ -60,6 +81,10 @@ pipeline { timestamps () } + parameters { + string(name: 'CART_BRANCH', defaultValue: '', + description: "Which cart job (PR, branch, etc.) to use for the build and test") + } stages { stage('Cancel Previous Builds') { when { changeRequest() } @@ -75,7 +100,13 @@ pipeline { filename 'Dockerfile.centos:7' dir 'utils/docker' label 'docker_runner' - additionalBuildArgs "-t ${sanitized_JOB_NAME}-centos7 " + '$BUILDARGS' + additionalBuildArgs "-t ${sanitized_JOB_NAME}-centos7 " + + '$BUILDARGS --build-arg USE_RPMS=true ' + + get_deps_build_vars() + + '--build-arg CART_BRANCH=' + + params.CART_BRANCH + + ' --build-arg CART_VERSION=' + + get_timestamp() } } steps { @@ -103,7 +134,13 @@ pipeline { filename 'Dockerfile.centos:7' dir 'utils/docker' label 'docker_runner' - additionalBuildArgs "-t ${sanitized_JOB_NAME}-centos7 " + '$BUILDARGS' + additionalBuildArgs "-t ${sanitized_JOB_NAME}-centos7 " + + '$BUILDARGS --build-arg USE_RPMS=true ' + + get_deps_build_vars() + + '--build-arg CART_BRANCH=' + + params.CART_BRANCH + + ' --build-arg CART_VERSION=' + + get_timestamp() } } steps { @@ -212,6 +249,13 @@ pipeline { trap 'set +e; set -x; ssh -i ci_key jenkins@\$NODE "set -ex; sudo umount \$IOF_BASE"' EXIT ssh -i ci_key jenkins@\$NODE "set -x set -e + for ext in cart; do + sudo bash << EOF + yum-config-manager --add-repo=https://build.hpdd.intel.com/job/daos-stack/job/\\\${ext}/job/${params.CART_BRANCH}/lastSuccessfulBuild/artifact/artifacts/ + echo \"gpgcheck = False\" >> /etc/yum.repos.d/build.hpdd.intel.com_job_daos-stack_job_\\\${ext}_job_${params.CART_BRANCH}_lastSuccessfulBuild_artifact_artifacts_.repo +EOF + done + sudo yum -y install cart sudo mkdir -p \$IOF_BASE sudo mount -t nfs \$HOSTNAME:\$PWD \$IOF_BASE cd \$IOF_BASE @@ -304,6 +348,13 @@ pipeline { trap 'set +e; set -x; ssh -i ci_key jenkins@\$NODE "set -ex; sudo umount \$IOF_BASE"' EXIT ssh -i ci_key jenkins@\$NODE "set -x set -e + for ext in cart; do + sudo bash << EOF + yum-config-manager --add-repo=https://build.hpdd.intel.com/job/daos-stack/job/\\\${ext}/job/${params.CART_BRANCH}/lastSuccessfulBuild/artifact/artifacts/ + echo \"gpgcheck = False\" >> /etc/yum.repos.d/build.hpdd.intel.com_job_daos-stack_job_\\\${ext}_job_${params.CART_BRANCH}_lastSuccessfulBuild_artifact_artifacts_.repo +EOF + done + sudo yum -y install cart sudo mkdir -p \$IOF_BASE sudo mount -t nfs \$HOSTNAME:\$PWD \$IOF_BASE cd \$IOF_BASE @@ -362,6 +413,13 @@ pipeline { trap 'set +e; set -x; ssh -i ci_key jenkins@\$NODE "set -ex; sudo umount \$IOF_BASE"' EXIT ssh -i ci_key jenkins@\$NODE "set -x set -e + for ext in cart; do + sudo bash << EOF + yum-config-manager --add-repo=https://build.hpdd.intel.com/job/daos-stack/job/\\\${ext}/job/${params.CART_BRANCH}/lastSuccessfulBuild/artifact/artifacts/ + echo \"gpgcheck = False\" >> /etc/yum.repos.d/build.hpdd.intel.com_job_daos-stack_job_\\\${ext}_job_${params.CART_BRANCH}_lastSuccessfulBuild_artifact_artifacts_.repo +EOF + done + sudo yum -y install cart sudo mkdir -p \$IOF_BASE sudo mount -t nfs \$HOSTNAME:\$PWD \$IOF_BASE cd \$IOF_BASE diff --git a/utils/docker/Dockerfile.centos:7 b/utils/docker/Dockerfile.centos:7 index 0813077c..d3a1b273 100644 --- a/utils/docker/Dockerfile.centos:7 +++ b/utils/docker/Dockerfile.centos:7 @@ -43,31 +43,56 @@ FROM centos:7 MAINTAINER Johann Lombardi -# Build arguments can be set via -build-arg +# Build arguments can be set via --build-arg # use same UID as host and default value of 1000 if not specified ARG UID=1000 +ARG USE_RPMS=false # for good measure, clean the metadata RUN yum clean metadata # Install basic tools RUN yum -y install epel-release -RUN yum -y install git gcc gcc-c++ make cmake golang libtool scons boost-devel \ - libuuid-devel openssl-devel libevent-devel libtool-ltdl-devel \ - libcmocka libcmocka-devel readline-devel \ - doxygen pandoc flex patch nasm yasm ninja-build meson \ - CUnit-devel libaio-devel python-pep8 lcov \ - python clang-analyzer sg3_utils libiscsi-devel \ - numactl-devel doxygen graphviz pylint python2-pygithub \ - python34 python34-pylint python34-PyYAML.x86_64 PyYAML \ - python2-paramiko python34-paramiko valgrind rsync \ - libyaml-devel file ShellCheck yum-plugin-copr python34-nose \ - python34-pip fuse python34-devel python-devel +RUN yum -y install \ + git gcc gcc-c++ make cmake golang libtool boost-devel \ + libuuid-devel openssl-devel libevent-devel hwloc-devel \ + libtool-ltdl-devel libcmocka-devel readline-devel \ + doxygen pandoc flex patch nasm yasm ninja-build meson \ + CUnit-devel libaio-devel clang-analyzer sg3_utils \ + libiscsi-devel numactl-devel graphviz \ + libyaml-devel \ + scons pylint file ShellCheck python-pep8 lcov python \ + python2-pygithub yum-plugin-copr + +# external deps build/repos +# sadly this doesn't work +#RUN yum-config-manager --save --setopt=*_job_daos-stack_job_${ext}_job_master_lastSuccessfulBuild_artifact_artifacts_.gpgcheck=0 +RUN if ! $USE_RPMS; then exit 0; fi; \ +for ext in openpa libfabric mercury pmix ompi; do \ + yum-config-manager --add-repo=https://build.hpdd.intel.com/job/daos-stack/job/${ext}/job/master/lastSuccessfulBuild/artifact/artifacts/; \ + echo "gpgcheck = False" >> /etc/yum.repos.d/build.hpdd.intel.com_job_daos-stack_job_${ext}_job_master_lastSuccessfulBuild_artifact_artifacts_.repo; \ +done + +ARG OPENPA=unknown +ARG LIBFABRIC=unknown +ARG MERCURY=unknown +ARG PMIX=unknown +ARG OMPI=unknown + +RUN if ! $USE_RPMS; then exit 0; fi; \ +OPENPA=${OPENPA} \ +LIBFABRIC=${LIBFABRIC} \ +MERCURY=${MERCURY} \ +PMIX=${PMIX} \ +OMPI=${OMPI} \ +yum -y install openpa-devel libfabric-devel mercury-devel pmix-devel ompi-devel + RUN yum -y copr enable jhli/ipmctl RUN yum -y copr enable jhli/safeclib -RUN yum -y install libsafec libipmctl-devel +RUN yum -y install libipmctl-devel -RUN pip3.4 install tabulate +RUN yum -y install doxygen python34 python34-pylint python34-PyYAML.x86_64 \ + PyYAML python2-paramiko python34-paramiko valgrind rsync # Add CaRT user ENV USER cart @@ -75,6 +100,23 @@ ENV PASSWD cart RUN useradd -u $UID -ms /bin/bash $USER RUN echo "$USER:$PASSWD" | chpasswd +RUN yum -y install python34-nose python34-pip fuse python34-devel python-devel + +RUN pip3.4 install tabulate + +ARG CART_BRANCH=unknown + +RUN if ! $USE_RPMS; then exit 0; fi; \ +for ext in cart; do \ + yum-config-manager --add-repo=https://build.hpdd.intel.com/job/daos-stack/job/${ext}/job/${CART_BRANCH}/lastSuccessfulBuild/artifact/artifacts/; \ + echo "gpgcheck = False" >> /etc/yum.repos.d/build.hpdd.intel.com_job_daos-stack_job_${ext}_job_${CART_BRANCH}_lastSuccessfulBuild_artifact_artifacts_.repo; \ +done + +ARG CART_VERSION=unknown + +RUN if ! $USE_RPMS; then exit 0; fi; \ +CART_VERSION=${CART_VERSION} yum -y install cart + # Switch to new user USER $USER WORKDIR /home/$USER From aee78e2ea3022c7e51233ed2c84e1b99ff4ba44d Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Wed, 20 Mar 2019 12:50:40 -0400 Subject: [PATCH 05/16] Only run RPM build and test when being run a cart PR For now. Signed-off-by: Brian J. Murrell --- Jenkinsfile | 287 +++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 238 insertions(+), 49 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 660b2b12..a601afc6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -85,6 +85,7 @@ pipeline { string(name: 'CART_BRANCH', defaultValue: '', description: "Which cart job (PR, branch, etc.) to use for the build and test") } + stages { stage('Cancel Previous Builds') { when { changeRequest() } @@ -101,12 +102,7 @@ pipeline { dir 'utils/docker' label 'docker_runner' additionalBuildArgs "-t ${sanitized_JOB_NAME}-centos7 " + - '$BUILDARGS --build-arg USE_RPMS=true ' + - get_deps_build_vars() + - '--build-arg CART_BRANCH=' + - params.CART_BRANCH + - ' --build-arg CART_VERSION=' + - get_timestamp() + '$BUILDARGS --build-arg USE_RPMS=false' } } steps { @@ -129,6 +125,39 @@ pipeline { //failFast true parallel { stage('Build on CentOS 7') { + when { expression { params.CART_BRANCH == '' } } + agent { + dockerfile { + filename 'Dockerfile.centos:7' + dir 'utils/docker' + label 'docker_runner' + additionalBuildArgs "-t ${sanitized_JOB_NAME}-centos7 " + + '$BUILDARGS --build-arg USE_RPMS=false' + } + } + steps { + sconsBuild clean: '_build.external iof.conf' + stash name: 'CentOS-install', includes: 'install/**' + stash name: 'CentOS-build-vars', includes: '.build_vars.*' + } + post { + always { + node('lightweight') { + recordIssues enabledForFailure: true, + aggregatingResults: true, + id: "analysis-centos7", + tools: [ gcc4(), cppCheck() ], + filters: [excludeFile(".*\\/_build\\.external\\/.*"), + excludeFile("_build\\.external\\/.*")] + } + } + success { + sh "rm -rf _build.external" + } + } + } + stage('Build on CentOS 7 with RPMs') { + when { expression { params.CART_BRANCH != '' } } agent { dockerfile { filename 'Dockerfile.centos:7' @@ -144,9 +173,7 @@ pipeline { } } steps { - sconsBuild(clean: '_build.external iof.conf') - stash name: 'CentOS-install', includes: 'install/**' - stash name: 'CentOS-build-vars', includes: '.build_vars.*' + sconsBuild clean: '_build.external iof.conf' } post { always { @@ -175,8 +202,8 @@ pipeline { } } steps { - sconsBuild(clean: '_build.external iof.conf', - scons_args: '--build-config=utils/build-master.config') + sconsBuild clean: '_build.external iof.conf', + scons_args: '--build-config=utils/build-master.config' stash name: 'CentOS-master-install', includes: 'install/**' stash name: 'CentOS-master-build-vars', includes: ".build_vars.*" } @@ -198,6 +225,7 @@ pipeline { } } stage('Build on Ubuntu 18.04') { + when { expression { params.CART_BRANCH == '' } } agent { dockerfile { filename 'Dockerfile.ubuntu:18.04' @@ -230,6 +258,52 @@ pipeline { stage('Test') { parallel { stage('Single node') { + when { expression { params.CART_BRANCH == '' } } + agent { + label 'ci_vm1' + } + options { + timeout(time: 60, unit: 'MINUTES') + } + steps { + provisionNodes NODELIST: env.NODELIST, + node_count: 1, + snapshot: true + runTest stashes: [ 'CentOS-install', 'CentOS-build-vars' ], + script: """set -x + . ./.build_vars.sh + IOF_BASE=\${SL_PREFIX%/install*} + NODELIST=$nodelist + NODE=\${NODELIST%%,*} + trap 'set +e; set -x; ssh -i ci_key jenkins@\$NODE "set -ex; sudo umount \$IOF_BASE"' EXIT + ssh -i ci_key jenkins@\$NODE "set -x + set -e + sudo mkdir -p \$IOF_BASE + sudo mount -t nfs \$HOSTNAME:\$PWD \$IOF_BASE + cd \$IOF_BASE + ln -s /usr/bin/fusermount install/bin/fusermount3 + pip3.4 install --user tabulate + export TR_USE_VALGRIND=none + export IOF_TESTLOG=test/output-centos + nosetests-3.4 --xunit-testsuite-name=centos --xunit-file=nosetests-centos.xml --exe --with-xunit" + exit 0 + """, + junit_files: 'nosetests-centos.xml' + } + post { + always { + junit 'nosetests-centos.xml' + archiveArtifacts artifacts: '**/*.log' + } + cleanup { + dir('test/output-centos') { + deleteDir() + } + } + } + } + stage('Single node with RPMs') { + when { expression { params.CART_BRANCH != '' } } agent { label 'ci_vm1' } @@ -263,14 +337,14 @@ EOF pip3.4 install --user tabulate export TR_USE_VALGRIND=none export IOF_TESTLOG=test/output-centos - nosetests-3.4 --xunit-testsuite-name=centos --xunit-file=nosetests-centos.xml --exe --with-xunit" + nosetests-3.4 --xunit-testsuite-name=centos --xunit-file=nosetests-centos-rpms.xml --exe --with-xunit" exit 0 """, - junit_files: 'nosetests-centos.xml' + junit_files: 'nosetests-centos-rpms.xml' } post { always { - junit 'nosetests-centos.xml' + junit 'nosetests-centos-rpms.xml' archiveArtifacts artifacts: '**/*.log' } cleanup { @@ -329,6 +403,66 @@ EOF } } stage('Single node valgrind') { + when { expression { params.CART_BRANCH == '' } } + agent { + label 'ci_vm1' + } + options { + timeout(time: 60, unit: 'MINUTES') + } + steps { + provisionNodes NODELIST: env.NODELIST, + node_count: 1, + snapshot: true + runTest stashes: [ 'CentOS-install', 'CentOS-build-vars' ], + script: """set -x + . ./.build_vars.sh + IOF_BASE=\${SL_PREFIX%/install*} + NODELIST=$nodelist + NODE=\${NODELIST%%,*} + trap 'set +e; set -x; ssh -i ci_key jenkins@\$NODE "set -ex; sudo umount \$IOF_BASE"' EXIT + ssh -i ci_key jenkins@\$NODE "set -x + set -e + sudo mkdir -p \$IOF_BASE + sudo mount -t nfs \$HOSTNAME:\$PWD \$IOF_BASE + cd \$IOF_BASE + ln -s /usr/bin/fusermount install/bin/fusermount3 + pip3.4 install --user tabulate + export TR_USE_VALGRIND=memcheck + export IOF_TESTLOG=test/output-memcheck + nosetests-3.4 --xunit-testsuite-name=valgrind --xunit-file=nosetests-valgrind.xml --exe --with-xunit" + exit 0 + """, + junit_files: 'nosetests-valgrind.xml' + } + post { + always { + junit 'nosetests-valgrind.xml' + archiveArtifacts artifacts: '**/*.log,**/*.memcheck' + publishValgrind ( + failBuildOnInvalidReports: true, + failBuildOnMissingReports: true, + failThresholdDefinitelyLost: '0', + failThresholdInvalidReadWrite: '0', + failThresholdTotal: '0', + pattern: '**/*.memcheck', + publishResultsForAbortedBuilds: false, + publishResultsForFailedBuilds: false, + sourceSubstitutionPaths: '', + unstableThresholdDefinitelyLost: '', + unstableThresholdInvalidReadWrite: '', + unstableThresholdTotal: '' + ) + } + cleanup { + dir('test/output-memcheck') { + deleteDir() + } + } + } + } + stage('Single node valgrind with RPMs') { + when { expression { params.CART_BRANCH != '' } } agent { label 'ci_vm1' } @@ -362,14 +496,14 @@ EOF pip3.4 install --user tabulate export TR_USE_VALGRIND=memcheck export IOF_TESTLOG=test/output-memcheck - nosetests-3.4 --xunit-testsuite-name=valgrind --xunit-file=nosetests-valgrind.xml --exe --with-xunit" + nosetests-3.4 --xunit-testsuite-name=valgrind --xunit-file=nosetests-valgrind-rpms.xml --exe --with-xunit" exit 0 """, - junit_files: 'nosetests-valgrind.xml' + junit_files: 'nosetests-valgrind-rpms.xml' } post { always { - junit 'nosetests-valgrind.xml' + junit 'nosetests-valgrind-rpms.xml' archiveArtifacts artifacts: '**/*.log,**/*.memcheck' publishValgrind ( failBuildOnInvalidReports: true, @@ -393,40 +527,95 @@ EOF } } } - stage('Fault injection') { - agent { - label 'ci_vm1' - } - options { - timeout(time: 60, unit: 'MINUTES') + stage('Fault injection') { + when { expression { params.CART_BRANCH == '' } } + agent { + label 'ci_vm1' + } + options { + timeout(time: 60, unit: 'MINUTES') + } + steps { + provisionNodes NODELIST: env.NODELIST, + node_count: 1, + snapshot: true + runTest stashes: [ 'CentOS-install', 'CentOS-build-vars' ], + script: """set -x + . ./.build_vars.sh + IOF_BASE=\${SL_PREFIX%/install*} + NODELIST=$nodelist + NODE=\${NODELIST%%,*} + trap 'set +e; set -x; ssh -i ci_key jenkins@\$NODE "set -ex; sudo umount \$IOF_BASE"' EXIT + ssh -i ci_key jenkins@\$NODE "set -x + set -e + sudo mkdir -p \$IOF_BASE + sudo mount -t nfs \$HOSTNAME:\$PWD \$IOF_BASE + cd \$IOF_BASE + ln -s /usr/bin/fusermount install/bin/fusermount3 + pip3.4 install --user tabulate + ./test/iof_test_alloc_fail.py" + """ + } + post { + always { + archiveArtifacts artifacts: '**/*.log,**/*.memcheck' + publishValgrind ( + failBuildOnInvalidReports: true, + failBuildOnMissingReports: true, + failThresholdDefinitelyLost: '0', + failThresholdInvalidReadWrite: '0', + failThresholdTotal: '0', + pattern: '**/*.memcheck', + publishResultsForAbortedBuilds: false, + publishResultsForFailedBuilds: false, + sourceSubstitutionPaths: '', + unstableThresholdDefinitelyLost: '', + unstableThresholdInvalidReadWrite: '', + unstableThresholdTotal: '' + ) + } + cleanup { + dir('test/output') { + deleteDir() + } + } + } } - steps { - provisionNodes NODELIST: env.NODELIST, - node_count: 1, - snapshot: true - runTest stashes: [ 'CentOS-install', 'CentOS-build-vars' ], - script: """set -x - . ./.build_vars.sh - IOF_BASE=\${SL_PREFIX%/install*} - NODELIST=$nodelist - NODE=\${NODELIST%%,*} - trap 'set +e; set -x; ssh -i ci_key jenkins@\$NODE "set -ex; sudo umount \$IOF_BASE"' EXIT - ssh -i ci_key jenkins@\$NODE "set -x - set -e - for ext in cart; do - sudo bash << EOF - yum-config-manager --add-repo=https://build.hpdd.intel.com/job/daos-stack/job/\\\${ext}/job/${params.CART_BRANCH}/lastSuccessfulBuild/artifact/artifacts/ - echo \"gpgcheck = False\" >> /etc/yum.repos.d/build.hpdd.intel.com_job_daos-stack_job_\\\${ext}_job_${params.CART_BRANCH}_lastSuccessfulBuild_artifact_artifacts_.repo + stage('Fault injection with RPMs') { + when { expression { params.CART_BRANCH != '' } } + agent { + label 'ci_vm1' + } + options { + timeout(time: 60, unit: 'MINUTES') + } + steps { + provisionNodes NODELIST: env.NODELIST, + node_count: 1, + snapshot: true + runTest stashes: [ 'CentOS-install', 'CentOS-build-vars' ], + script: """set -x + . ./.build_vars.sh + IOF_BASE=\${SL_PREFIX%/install*} + NODELIST=$nodelist + NODE=\${NODELIST%%,*} + trap 'set +e; set -x; ssh -i ci_key jenkins@\$NODE "set -ex; sudo umount \$IOF_BASE"' EXIT + ssh -i ci_key jenkins@\$NODE "set -x + set -e + for ext in cart; do + sudo bash << EOF + yum-config-manager --add-repo=https://build.hpdd.intel.com/job/daos-stack/job/\\\${ext}/job/${params.CART_BRANCH}/lastSuccessfulBuild/artifact/artifacts/ + echo \"gpgcheck = False\" >> /etc/yum.repos.d/build.hpdd.intel.com_job_daos-stack_job_\\\${ext}_job_${params.CART_BRANCH}_lastSuccessfulBuild_artifact_artifacts_.repo EOF - done - sudo yum -y install cart - sudo mkdir -p \$IOF_BASE - sudo mount -t nfs \$HOSTNAME:\$PWD \$IOF_BASE - cd \$IOF_BASE - ln -s /usr/bin/fusermount install/bin/fusermount3 - pip3.4 install --user tabulate - ./test/iof_test_alloc_fail.py" - """ + done + sudo yum -y install cart + sudo mkdir -p \$IOF_BASE + sudo mount -t nfs \$HOSTNAME:\$PWD \$IOF_BASE + cd \$IOF_BASE + ln -s /usr/bin/fusermount install/bin/fusermount3 + pip3.4 install --user tabulate + ./test/iof_test_alloc_fail.py" + """ } post { always { From a07f713d30f14992776c8413a368b9254a86b26c Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Wed, 20 Mar 2019 13:14:55 -0400 Subject: [PATCH 06/16] Pipeline native when conditions Signed-off-by: Brian J. Murrell --- Jenkinsfile | 45 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 36 insertions(+), 9 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index a601afc6..536cb30e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -125,7 +125,10 @@ pipeline { //failFast true parallel { stage('Build on CentOS 7') { - when { expression { params.CART_BRANCH == '' } } + when { + beforeAgent true + equals expected: '', actual: params.CART_BRANCH + } agent { dockerfile { filename 'Dockerfile.centos:7' @@ -157,7 +160,10 @@ pipeline { } } stage('Build on CentOS 7 with RPMs') { - when { expression { params.CART_BRANCH != '' } } + when { + beforeAgent true + not { equals expected: '', actual: params.CART_BRANCH } + } agent { dockerfile { filename 'Dockerfile.centos:7' @@ -225,7 +231,10 @@ pipeline { } } stage('Build on Ubuntu 18.04') { - when { expression { params.CART_BRANCH == '' } } + when { + beforeAgent true + equals expected: '', actual: params.CART_BRANCH + } agent { dockerfile { filename 'Dockerfile.ubuntu:18.04' @@ -258,7 +267,10 @@ pipeline { stage('Test') { parallel { stage('Single node') { - when { expression { params.CART_BRANCH == '' } } + when { + beforeAgent true + equals expected: '', actual: params.CART_BRANCH + } agent { label 'ci_vm1' } @@ -303,7 +315,10 @@ pipeline { } } stage('Single node with RPMs') { - when { expression { params.CART_BRANCH != '' } } + when { + beforeAgent true + not { equals expected: '', actual: params.CART_BRANCH } + } agent { label 'ci_vm1' } @@ -403,7 +418,10 @@ EOF } } stage('Single node valgrind') { - when { expression { params.CART_BRANCH == '' } } + when { + beforeAgent true + equals expected: '', actual: params.CART_BRANCH + } agent { label 'ci_vm1' } @@ -462,7 +480,10 @@ EOF } } stage('Single node valgrind with RPMs') { - when { expression { params.CART_BRANCH != '' } } + when { + beforeAgent true + not { equals expected: '', actual: params.CART_BRANCH } + } agent { label 'ci_vm1' } @@ -528,7 +549,10 @@ EOF } } stage('Fault injection') { - when { expression { params.CART_BRANCH == '' } } + when { + beforeAgent true + equals expected: '', actual: params.CART_BRANCH + } agent { label 'ci_vm1' } @@ -582,7 +606,10 @@ EOF } } stage('Fault injection with RPMs') { - when { expression { params.CART_BRANCH != '' } } + when { + beforeAgent true + not { equals expected: '', actual: params.CART_BRANCH } + } agent { label 'ci_vm1' } From c1a558f6b749880b7d5313480628da6e04aac5c3 Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Wed, 20 Mar 2019 14:13:26 -0400 Subject: [PATCH 07/16] Add beforeAgent true to cart-master build/test Signed-off-by: Brian J. Murrell --- Jenkinsfile | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 536cb30e..f65f5b5d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -198,7 +198,10 @@ pipeline { } } stage('Build master CentOS 7') { - when { branch 'master' } + when { + beforeAgent true + branch 'master' + } agent { dockerfile { filename 'Dockerfile.centos:7' @@ -370,7 +373,10 @@ EOF } } stage('Single node cart-master') { - when { branch 'master' } + when { + beforeAgent true + branch 'master' + } agent { label 'ci_vm1' } From ea095083a204a04ea7a61bd6e7fe8f6705d05995 Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Wed, 20 Mar 2019 14:10:12 -0400 Subject: [PATCH 08/16] Stash build for RPMs build too Signed-off-by: Brian J. Murrell --- Jenkinsfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index f65f5b5d..8ef1fde8 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -180,6 +180,8 @@ pipeline { } steps { sconsBuild clean: '_build.external iof.conf' + stash name: 'CentOS-install', includes: 'install/**' + stash name: 'CentOS-build-vars', includes: '.build_vars.*' } post { always { From 0896871d3889aaee24464a3767e47de5fb70e1ce Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Wed, 20 Mar 2019 12:11:53 -0400 Subject: [PATCH 09/16] Skip Ubuntu for quicker turn-around --- Jenkinsfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 8ef1fde8..3e2ec5fe 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -249,7 +249,8 @@ pipeline { } } steps { - sconsBuild(clean: '_build.external iof.conf') + echo "Skipping" + //sconsBuild clean: "_build.external iof.conf" } post { always { From d7dabeb19cfb505555049c8d4be63246e514aa07 Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Tue, 19 Mar 2019 11:50:48 -0400 Subject: [PATCH 10/16] Temporary changes until everything is landed Signed-off-by: Brian J. Murrell --- Jenkinsfile | 17 ++++++++++------- utils/docker/Dockerfile.centos:7 | 8 ++++---- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 3e2ec5fe..40a6832d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -36,6 +36,9 @@ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +// To use a test branch (i.e. PR) until it lands to master +// I.e. for testing library changes +@Library(value="pipeline-lib@pipeline-pr-test-rpms") _ def sanitized_JOB_NAME = JOB_NAME.toLowerCase().replaceAll('/', '-').replaceAll('%2f', '-') @@ -82,7 +85,7 @@ pipeline { } parameters { - string(name: 'CART_BRANCH', defaultValue: '', + string(name: 'CART_BRANCH', defaultValue: 'PR-93', description: "Which cart job (PR, branch, etc.) to use for the build and test") } @@ -346,8 +349,8 @@ pipeline { set -e for ext in cart; do sudo bash << EOF - yum-config-manager --add-repo=https://build.hpdd.intel.com/job/daos-stack/job/\\\${ext}/job/${params.CART_BRANCH}/lastSuccessfulBuild/artifact/artifacts/ - echo \"gpgcheck = False\" >> /etc/yum.repos.d/build.hpdd.intel.com_job_daos-stack_job_\\\${ext}_job_${params.CART_BRANCH}_lastSuccessfulBuild_artifact_artifacts_.repo + yum-config-manager --add-repo=https://build.hpdd.intel.com/job/daos-stack/job/\\\${ext}/job/${params.CART_BRANCH}/lastCompletedBuild/artifact/artifacts/ + echo \"gpgcheck = False\" >> /etc/yum.repos.d/build.hpdd.intel.com_job_daos-stack_job_\\\${ext}_job_${params.CART_BRANCH}_lastCompletedBuild_artifact_artifacts_.repo EOF done sudo yum -y install cart @@ -514,8 +517,8 @@ EOF set -e for ext in cart; do sudo bash << EOF - yum-config-manager --add-repo=https://build.hpdd.intel.com/job/daos-stack/job/\\\${ext}/job/${params.CART_BRANCH}/lastSuccessfulBuild/artifact/artifacts/ - echo \"gpgcheck = False\" >> /etc/yum.repos.d/build.hpdd.intel.com_job_daos-stack_job_\\\${ext}_job_${params.CART_BRANCH}_lastSuccessfulBuild_artifact_artifacts_.repo + yum-config-manager --add-repo=https://build.hpdd.intel.com/job/daos-stack/job/\\\${ext}/job/${params.CART_BRANCH}/lastCompletedBuild/artifact/artifacts/ + echo \"gpgcheck = False\" >> /etc/yum.repos.d/build.hpdd.intel.com_job_daos-stack_job_\\\${ext}_job_${params.CART_BRANCH}_lastCompletedBuild_artifact_artifacts_.repo EOF done sudo yum -y install cart @@ -640,8 +643,8 @@ EOF set -e for ext in cart; do sudo bash << EOF - yum-config-manager --add-repo=https://build.hpdd.intel.com/job/daos-stack/job/\\\${ext}/job/${params.CART_BRANCH}/lastSuccessfulBuild/artifact/artifacts/ - echo \"gpgcheck = False\" >> /etc/yum.repos.d/build.hpdd.intel.com_job_daos-stack_job_\\\${ext}_job_${params.CART_BRANCH}_lastSuccessfulBuild_artifact_artifacts_.repo + yum-config-manager --add-repo=https://build.hpdd.intel.com/job/daos-stack/job/\\\${ext}/job/${params.CART_BRANCH}/lastCompletedBuild/artifact/artifacts/ + echo \"gpgcheck = False\" >> /etc/yum.repos.d/build.hpdd.intel.com_job_daos-stack_job_\\\${ext}_job_${params.CART_BRANCH}_lastCompletedBuild_artifact_artifacts_.repo EOF done sudo yum -y install cart diff --git a/utils/docker/Dockerfile.centos:7 b/utils/docker/Dockerfile.centos:7 index d3a1b273..d9508d2f 100644 --- a/utils/docker/Dockerfile.centos:7 +++ b/utils/docker/Dockerfile.centos:7 @@ -106,10 +106,10 @@ RUN pip3.4 install tabulate ARG CART_BRANCH=unknown -RUN if ! $USE_RPMS; then exit 0; fi; \ -for ext in cart; do \ - yum-config-manager --add-repo=https://build.hpdd.intel.com/job/daos-stack/job/${ext}/job/${CART_BRANCH}/lastSuccessfulBuild/artifact/artifacts/; \ - echo "gpgcheck = False" >> /etc/yum.repos.d/build.hpdd.intel.com_job_daos-stack_job_${ext}_job_${CART_BRANCH}_lastSuccessfulBuild_artifact_artifacts_.repo; \ +RUN if ! $USE_RPMS; then exit 0; fi; \ +for ext in cart; do \ + yum-config-manager --add-repo=https://build.hpdd.intel.com/job/daos-stack/job/${ext}/job/${CART_BRANCH}/lastCompletedBuild/artifact/artifacts/; \ + echo "gpgcheck = False" >> /etc/yum.repos.d/build.hpdd.intel.com_job_daos-stack_job_${ext}_job_${CART_BRANCH}_lastCompletedBuild_artifact_artifacts_.repo; \ done ARG CART_VERSION=unknown From 5801809ea9de628eba7fbcdb0ed6ee5ebe641563 Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Wed, 20 Mar 2019 12:07:59 -0400 Subject: [PATCH 11/16] Update to supporting scons_local Signed-off-by: Brian J. Murrell --- scons_local | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scons_local b/scons_local index 9b28b014..0ccf6c15 160000 --- a/scons_local +++ b/scons_local @@ -1 +1 @@ -Subproject commit 9b28b014328221be2c27dcca8a3e82f5dff46031 +Subproject commit 0ccf6c15881047f2f331d76a7d720c9fe7aa0ae1 From a08b79603a0628f3f0f1153412cb0cd415b29deb Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Fri, 22 Mar 2019 11:53:35 -0400 Subject: [PATCH 12/16] Move scons_local Signed-off-by: Brian J. Murrell --- scons_local | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scons_local b/scons_local index 0ccf6c15..49c6d12d 160000 --- a/scons_local +++ b/scons_local @@ -1 +1 @@ -Subproject commit 0ccf6c15881047f2f331d76a7d720c9fe7aa0ae1 +Subproject commit 49c6d12d182a4444f1c10586d9ce0813bfdc1870 From ee02bad12e52a1a9e47156a6fb93032c68776bab Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Thu, 28 Mar 2019 12:54:01 -0400 Subject: [PATCH 13/16] Remove duplicate doxygen install Signed-off-by: Brian J. Murrell --- utils/docker/Dockerfile.centos:7 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/docker/Dockerfile.centos:7 b/utils/docker/Dockerfile.centos:7 index d9508d2f..7d8a3d0a 100644 --- a/utils/docker/Dockerfile.centos:7 +++ b/utils/docker/Dockerfile.centos:7 @@ -91,7 +91,7 @@ RUN yum -y copr enable jhli/ipmctl RUN yum -y copr enable jhli/safeclib RUN yum -y install libipmctl-devel -RUN yum -y install doxygen python34 python34-pylint python34-PyYAML.x86_64 \ +RUN yum -y install python34 python34-pylint python34-PyYAML.x86_64 \ PyYAML python2-paramiko python34-paramiko valgrind rsync # Add CaRT user From faf36d1d2a53713b84e18236ee59160446d2238c Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Thu, 28 Mar 2019 13:00:44 -0400 Subject: [PATCH 14/16] Harmonize Dockerfile with scons_local Signed-off-by: Brian J. Murrell --- utils/docker/Dockerfile.centos:7 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/utils/docker/Dockerfile.centos:7 b/utils/docker/Dockerfile.centos:7 index 7d8a3d0a..4ba8eba8 100644 --- a/utils/docker/Dockerfile.centos:7 +++ b/utils/docker/Dockerfile.centos:7 @@ -55,7 +55,7 @@ RUN yum clean metadata RUN yum -y install epel-release RUN yum -y install \ git gcc gcc-c++ make cmake golang libtool boost-devel \ - libuuid-devel openssl-devel libevent-devel hwloc-devel \ + libuuid-devel openssl-devel libevent-devel \ libtool-ltdl-devel libcmocka-devel readline-devel \ doxygen pandoc flex patch nasm yasm ninja-build meson \ CUnit-devel libaio-devel clang-analyzer sg3_utils \ @@ -68,6 +68,7 @@ RUN yum -y install \ # sadly this doesn't work #RUN yum-config-manager --save --setopt=*_job_daos-stack_job_${ext}_job_master_lastSuccessfulBuild_artifact_artifacts_.gpgcheck=0 RUN if ! $USE_RPMS; then exit 0; fi; \ +yum -y install hwloc-devel; \ for ext in openpa libfabric mercury pmix ompi; do \ yum-config-manager --add-repo=https://build.hpdd.intel.com/job/daos-stack/job/${ext}/job/master/lastSuccessfulBuild/artifact/artifacts/; \ echo "gpgcheck = False" >> /etc/yum.repos.d/build.hpdd.intel.com_job_daos-stack_job_${ext}_job_master_lastSuccessfulBuild_artifact_artifacts_.repo; \ From 83b874e4cd899736899dae662f74858c074f30e8 Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Thu, 28 Mar 2019 13:07:06 -0400 Subject: [PATCH 15/16] Updates Signed-off-by: Brian J. Murrell --- Jenkinsfile | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 40a6832d..d1cdc078 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -52,7 +52,8 @@ String get_deps_build_vars() { 'MERCURY' : '1.0.1-2', 'OMPI' : '3.0.0rc4-3', 'PMIX' : '2.1.1-2', - 'LIBFABRIC': '1.7.1rc1-1' + 'LIBFABRIC': '1.7.1rc1-1', + 'OPENPA' : '1.0.4-2' ] def buildargs = "" deps.each { @@ -128,7 +129,7 @@ pipeline { //failFast true parallel { stage('Build on CentOS 7') { - when { + when { beforeAgent true equals expected: '', actual: params.CART_BRANCH } @@ -142,7 +143,7 @@ pipeline { } } steps { - sconsBuild clean: '_build.external iof.conf' + sconsBuild clean: '_build.external' stash name: 'CentOS-install', includes: 'install/**' stash name: 'CentOS-build-vars', includes: '.build_vars.*' } @@ -182,7 +183,7 @@ pipeline { } } steps { - sconsBuild clean: '_build.external iof.conf' + sconsBuild clean: '_build.external' stash name: 'CentOS-install', includes: 'install/**' stash name: 'CentOS-build-vars', includes: '.build_vars.*' } @@ -216,7 +217,7 @@ pipeline { } } steps { - sconsBuild clean: '_build.external iof.conf', + sconsBuild clean: '_build.external', scons_args: '--build-config=utils/build-master.config' stash name: 'CentOS-master-install', includes: 'install/**' stash name: 'CentOS-master-build-vars', includes: ".build_vars.*" @@ -239,7 +240,7 @@ pipeline { } } stage('Build on Ubuntu 18.04') { - when { + when { beforeAgent true equals expected: '', actual: params.CART_BRANCH } @@ -253,7 +254,7 @@ pipeline { } steps { echo "Skipping" - //sconsBuild clean: "_build.external iof.conf" + //sconsBuild clean: "_build.external" } post { always { @@ -276,7 +277,7 @@ pipeline { stage('Test') { parallel { stage('Single node') { - when { + when { beforeAgent true equals expected: '', actual: params.CART_BRANCH } @@ -430,7 +431,7 @@ EOF } } stage('Single node valgrind') { - when { + when { beforeAgent true equals expected: '', actual: params.CART_BRANCH } @@ -561,7 +562,7 @@ EOF } } stage('Fault injection') { - when { + when { beforeAgent true equals expected: '', actual: params.CART_BRANCH } From 113a7fc29c2a5dc4670c6079ee4820e8c397c371 Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Thu, 28 Mar 2019 13:07:09 -0400 Subject: [PATCH 16/16] Move scons_local Signed-off-by: Brian J. Murrell --- scons_local | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scons_local b/scons_local index 49c6d12d..2df80716 160000 --- a/scons_local +++ b/scons_local @@ -1 +1 @@ -Subproject commit 49c6d12d182a4444f1c10586d9ce0813bfdc1870 +Subproject commit 2df80716a35de18da7a3e325522bae26514b15d1