diff --git a/Jenkinsfile b/Jenkinsfile index 2d4846db..d1cdc078 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -36,9 +36,34 @@ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +// To use a test branch (i.e. PR) until it lands to master +// I.e. for testing library changes +@Library(value="pipeline-lib@pipeline-pr-test-rpms") _ def sanitized_JOB_NAME = JOB_NAME.toLowerCase().replaceAll('/', '-').replaceAll('%2f', '-') +long get_timestamp() { + Date date = new Date() + return date.getTime() +} + +String get_deps_build_vars() { + def deps = [ + 'MERCURY' : '1.0.1-2', + 'OMPI' : '3.0.0rc4-3', + 'PMIX' : '2.1.1-2', + 'LIBFABRIC': '1.7.1rc1-1', + 'OPENPA' : '1.0.4-2' + ] + def buildargs = "" + deps.each { + dep, ver -> buildargs +="--build-arg ${dep}=" + + dep.toLowerCase() + "-${ver} " + } + + return buildargs +} + pipeline { agent { label 'lightweight' } @@ -60,6 +85,11 @@ pipeline { timestamps () } + parameters { + string(name: 'CART_BRANCH', defaultValue: 'PR-93', + description: "Which cart job (PR, branch, etc.) to use for the build and test") + } + stages { stage('Cancel Previous Builds') { when { changeRequest() } @@ -75,7 +105,8 @@ pipeline { filename 'Dockerfile.centos:7' dir 'utils/docker' label 'docker_runner' - additionalBuildArgs "-t ${sanitized_JOB_NAME}-centos7 " + '$BUILDARGS' + additionalBuildArgs "-t ${sanitized_JOB_NAME}-centos7 " + + '$BUILDARGS --build-arg USE_RPMS=false' } } steps { @@ -98,16 +129,61 @@ pipeline { //failFast true parallel { stage('Build on CentOS 7') { + when { + beforeAgent true + equals expected: '', actual: params.CART_BRANCH + } agent { dockerfile { filename 'Dockerfile.centos:7' dir 'utils/docker' label 'docker_runner' - additionalBuildArgs "-t ${sanitized_JOB_NAME}-centos7 " + '$BUILDARGS' + additionalBuildArgs "-t ${sanitized_JOB_NAME}-centos7 " + + '$BUILDARGS --build-arg USE_RPMS=false' + } + } + steps { + sconsBuild clean: '_build.external' + stash name: 'CentOS-install', includes: 'install/**' + stash name: 'CentOS-build-vars', includes: '.build_vars.*' + } + post { + always { + node('lightweight') { + recordIssues enabledForFailure: true, + aggregatingResults: true, + id: "analysis-centos7", + tools: [ gcc4(), cppCheck() ], + filters: [excludeFile(".*\\/_build\\.external\\/.*"), + excludeFile("_build\\.external\\/.*")] + } + } + success { + sh "rm -rf _build.external" + } + } + } + stage('Build on CentOS 7 with RPMs') { + when { + beforeAgent true + not { equals expected: '', actual: params.CART_BRANCH } + } + agent { + dockerfile { + filename 'Dockerfile.centos:7' + dir 'utils/docker' + label 'docker_runner' + additionalBuildArgs "-t ${sanitized_JOB_NAME}-centos7 " + + '$BUILDARGS --build-arg USE_RPMS=true ' + + get_deps_build_vars() + + '--build-arg CART_BRANCH=' + + params.CART_BRANCH + + ' --build-arg CART_VERSION=' + + get_timestamp() } } steps { - sconsBuild(clean: '_build.external iof.conf') + sconsBuild clean: '_build.external' stash name: 'CentOS-install', includes: 'install/**' stash name: 'CentOS-build-vars', includes: '.build_vars.*' } @@ -128,7 +204,10 @@ pipeline { } } stage('Build master CentOS 7') { - when { branch 'master' } + when { + beforeAgent true + branch 'master' + } agent { dockerfile { filename 'Dockerfile.centos:7' @@ -138,8 +217,8 @@ pipeline { } } steps { - sconsBuild(clean: '_build.external iof.conf', - scons_args: '--build-config=utils/build-master.config') + sconsBuild clean: '_build.external', + scons_args: '--build-config=utils/build-master.config' stash name: 'CentOS-master-install', includes: 'install/**' stash name: 'CentOS-master-build-vars', includes: ".build_vars.*" } @@ -161,6 +240,10 @@ pipeline { } } stage('Build on Ubuntu 18.04') { + when { + beforeAgent true + equals expected: '', actual: params.CART_BRANCH + } agent { dockerfile { filename 'Dockerfile.ubuntu:18.04' @@ -170,7 +253,8 @@ pipeline { } } steps { - sconsBuild(clean: '_build.external iof.conf') + echo "Skipping" + //sconsBuild clean: "_build.external" } post { always { @@ -193,6 +277,10 @@ pipeline { stage('Test') { parallel { stage('Single node') { + when { + beforeAgent true + equals expected: '', actual: params.CART_BRANCH + } agent { label 'ci_vm1' } @@ -236,8 +324,66 @@ pipeline { } } } + stage('Single node with RPMs') { + when { + beforeAgent true + not { equals expected: '', actual: params.CART_BRANCH } + } + agent { + label 'ci_vm1' + } + options { + timeout(time: 60, unit: 'MINUTES') + } + steps { + provisionNodes NODELIST: env.NODELIST, + node_count: 1, + snapshot: true + runTest stashes: [ 'CentOS-install', 'CentOS-build-vars' ], + script: """set -x + . ./.build_vars.sh + IOF_BASE=\${SL_PREFIX%/install*} + NODELIST=$nodelist + NODE=\${NODELIST%%,*} + trap 'set +e; set -x; ssh -i ci_key jenkins@\$NODE "set -ex; sudo umount \$IOF_BASE"' EXIT + ssh -i ci_key jenkins@\$NODE "set -x + set -e + for ext in cart; do + sudo bash << EOF + yum-config-manager --add-repo=https://build.hpdd.intel.com/job/daos-stack/job/\\\${ext}/job/${params.CART_BRANCH}/lastCompletedBuild/artifact/artifacts/ + echo \"gpgcheck = False\" >> /etc/yum.repos.d/build.hpdd.intel.com_job_daos-stack_job_\\\${ext}_job_${params.CART_BRANCH}_lastCompletedBuild_artifact_artifacts_.repo +EOF + done + sudo yum -y install cart + sudo mkdir -p \$IOF_BASE + sudo mount -t nfs \$HOSTNAME:\$PWD \$IOF_BASE + cd \$IOF_BASE + ln -s /usr/bin/fusermount install/bin/fusermount3 + pip3.4 install --user tabulate + export TR_USE_VALGRIND=none + export IOF_TESTLOG=test/output-centos + nosetests-3.4 --xunit-testsuite-name=centos --xunit-file=nosetests-centos-rpms.xml --exe --with-xunit" + exit 0 + """, + junit_files: 'nosetests-centos-rpms.xml' + } + post { + always { + junit 'nosetests-centos-rpms.xml' + archiveArtifacts artifacts: '**/*.log' + } + cleanup { + dir('test/output-centos') { + deleteDir() + } + } + } + } stage('Single node cart-master') { - when { branch 'master' } + when { + beforeAgent true + branch 'master' + } agent { label 'ci_vm1' } @@ -285,6 +431,10 @@ pipeline { } } stage('Single node valgrind') { + when { + beforeAgent true + equals expected: '', actual: params.CART_BRANCH + } agent { label 'ci_vm1' } @@ -342,33 +492,170 @@ pipeline { } } } - stage('Fault injection') { - agent { - label 'ci_vm1' + stage('Single node valgrind with RPMs') { + when { + beforeAgent true + not { equals expected: '', actual: params.CART_BRANCH } + } + agent { + label 'ci_vm1' + } + options { + timeout(time: 60, unit: 'MINUTES') + } + steps { + provisionNodes NODELIST: env.NODELIST, + node_count: 1, + snapshot: true + runTest stashes: [ 'CentOS-install', 'CentOS-build-vars' ], + script: """set -x + . ./.build_vars.sh + IOF_BASE=\${SL_PREFIX%/install*} + NODELIST=$nodelist + NODE=\${NODELIST%%,*} + trap 'set +e; set -x; ssh -i ci_key jenkins@\$NODE "set -ex; sudo umount \$IOF_BASE"' EXIT + ssh -i ci_key jenkins@\$NODE "set -x + set -e + for ext in cart; do + sudo bash << EOF + yum-config-manager --add-repo=https://build.hpdd.intel.com/job/daos-stack/job/\\\${ext}/job/${params.CART_BRANCH}/lastCompletedBuild/artifact/artifacts/ + echo \"gpgcheck = False\" >> /etc/yum.repos.d/build.hpdd.intel.com_job_daos-stack_job_\\\${ext}_job_${params.CART_BRANCH}_lastCompletedBuild_artifact_artifacts_.repo +EOF + done + sudo yum -y install cart + sudo mkdir -p \$IOF_BASE + sudo mount -t nfs \$HOSTNAME:\$PWD \$IOF_BASE + cd \$IOF_BASE + ln -s /usr/bin/fusermount install/bin/fusermount3 + pip3.4 install --user tabulate + export TR_USE_VALGRIND=memcheck + export IOF_TESTLOG=test/output-memcheck + nosetests-3.4 --xunit-testsuite-name=valgrind --xunit-file=nosetests-valgrind-rpms.xml --exe --with-xunit" + exit 0 + """, + junit_files: 'nosetests-valgrind-rpms.xml' + } + post { + always { + junit 'nosetests-valgrind-rpms.xml' + archiveArtifacts artifacts: '**/*.log,**/*.memcheck' + publishValgrind ( + failBuildOnInvalidReports: true, + failBuildOnMissingReports: true, + failThresholdDefinitelyLost: '0', + failThresholdInvalidReadWrite: '0', + failThresholdTotal: '0', + pattern: '**/*.memcheck', + publishResultsForAbortedBuilds: false, + publishResultsForFailedBuilds: false, + sourceSubstitutionPaths: '', + unstableThresholdDefinitelyLost: '', + unstableThresholdInvalidReadWrite: '', + unstableThresholdTotal: '' + ) + } + cleanup { + dir('test/output-memcheck') { + deleteDir() + } + } + } } - options { - timeout(time: 60, unit: 'MINUTES') + stage('Fault injection') { + when { + beforeAgent true + equals expected: '', actual: params.CART_BRANCH + } + agent { + label 'ci_vm1' + } + options { + timeout(time: 60, unit: 'MINUTES') + } + steps { + provisionNodes NODELIST: env.NODELIST, + node_count: 1, + snapshot: true + runTest stashes: [ 'CentOS-install', 'CentOS-build-vars' ], + script: """set -x + . ./.build_vars.sh + IOF_BASE=\${SL_PREFIX%/install*} + NODELIST=$nodelist + NODE=\${NODELIST%%,*} + trap 'set +e; set -x; ssh -i ci_key jenkins@\$NODE "set -ex; sudo umount \$IOF_BASE"' EXIT + ssh -i ci_key jenkins@\$NODE "set -x + set -e + sudo mkdir -p \$IOF_BASE + sudo mount -t nfs \$HOSTNAME:\$PWD \$IOF_BASE + cd \$IOF_BASE + ln -s /usr/bin/fusermount install/bin/fusermount3 + pip3.4 install --user tabulate + ./test/iof_test_alloc_fail.py" + """ + } + post { + always { + archiveArtifacts artifacts: '**/*.log,**/*.memcheck' + publishValgrind ( + failBuildOnInvalidReports: true, + failBuildOnMissingReports: true, + failThresholdDefinitelyLost: '0', + failThresholdInvalidReadWrite: '0', + failThresholdTotal: '0', + pattern: '**/*.memcheck', + publishResultsForAbortedBuilds: false, + publishResultsForFailedBuilds: false, + sourceSubstitutionPaths: '', + unstableThresholdDefinitelyLost: '', + unstableThresholdInvalidReadWrite: '', + unstableThresholdTotal: '' + ) + } + cleanup { + dir('test/output') { + deleteDir() + } + } + } } - steps { - provisionNodes NODELIST: env.NODELIST, - node_count: 1, - snapshot: true - runTest stashes: [ 'CentOS-install', 'CentOS-build-vars' ], - script: """set -x - . ./.build_vars.sh - IOF_BASE=\${SL_PREFIX%/install*} - NODELIST=$nodelist - NODE=\${NODELIST%%,*} - trap 'set +e; set -x; ssh -i ci_key jenkins@\$NODE "set -ex; sudo umount \$IOF_BASE"' EXIT - ssh -i ci_key jenkins@\$NODE "set -x - set -e - sudo mkdir -p \$IOF_BASE - sudo mount -t nfs \$HOSTNAME:\$PWD \$IOF_BASE - cd \$IOF_BASE - ln -s /usr/bin/fusermount install/bin/fusermount3 - pip3.4 install --user tabulate - ./test/iof_test_alloc_fail.py" - """ + stage('Fault injection with RPMs') { + when { + beforeAgent true + not { equals expected: '', actual: params.CART_BRANCH } + } + agent { + label 'ci_vm1' + } + options { + timeout(time: 60, unit: 'MINUTES') + } + steps { + provisionNodes NODELIST: env.NODELIST, + node_count: 1, + snapshot: true + runTest stashes: [ 'CentOS-install', 'CentOS-build-vars' ], + script: """set -x + . ./.build_vars.sh + IOF_BASE=\${SL_PREFIX%/install*} + NODELIST=$nodelist + NODE=\${NODELIST%%,*} + trap 'set +e; set -x; ssh -i ci_key jenkins@\$NODE "set -ex; sudo umount \$IOF_BASE"' EXIT + ssh -i ci_key jenkins@\$NODE "set -x + set -e + for ext in cart; do + sudo bash << EOF + yum-config-manager --add-repo=https://build.hpdd.intel.com/job/daos-stack/job/\\\${ext}/job/${params.CART_BRANCH}/lastCompletedBuild/artifact/artifacts/ + echo \"gpgcheck = False\" >> /etc/yum.repos.d/build.hpdd.intel.com_job_daos-stack_job_\\\${ext}_job_${params.CART_BRANCH}_lastCompletedBuild_artifact_artifacts_.repo +EOF + done + sudo yum -y install cart + sudo mkdir -p \$IOF_BASE + sudo mount -t nfs \$HOSTNAME:\$PWD \$IOF_BASE + cd \$IOF_BASE + ln -s /usr/bin/fusermount install/bin/fusermount3 + pip3.4 install --user tabulate + ./test/iof_test_alloc_fail.py" + """ } post { always { diff --git a/scons_local b/scons_local index 9b28b014..2df80716 160000 --- a/scons_local +++ b/scons_local @@ -1 +1 @@ -Subproject commit 9b28b014328221be2c27dcca8a3e82f5dff46031 +Subproject commit 2df80716a35de18da7a3e325522bae26514b15d1 diff --git a/test/common_methods.py b/test/common_methods.py index 18467a57..67c12e66 100644 --- a/test/common_methods.py +++ b/test/common_methods.py @@ -855,7 +855,8 @@ def test_self_test(self): self_test = find_executable('self_test') if not self_test: - cart_prefix = os.getenv("IOF_CART_PREFIX", None) + cart_prefix = os.getenv("IOF_CART_PREFIX", + iofcommontestsuite.CART_PREFIX) if not cart_prefix: self.skipTest('Could not find self_test binary') self_test = os.path.join(cart_prefix, 'bin', 'self_test') @@ -866,7 +867,7 @@ def test_self_test(self): environ['OFI_INTERFACE'] = self.ofi_interface cmd = [self_test, '--singleton', '--path', self.cnss_prefix, '--group-name', 'IONSS', '-e' '0:0', - '-r', '1000', '-s' '0 0,0 128,128 0'] + '-r', '50', '-s' '0 0,0 128,128 0'] log_top_dir = os.getenv("IOF_TESTLOG", os.path.join(os.path.dirname( @@ -875,8 +876,9 @@ def test_self_test(self): if not os.path.exists(log_path): os.makedirs(log_path) - cmdfileout = os.path.join(log_path, "self_test.out") - cmdfileerr = os.path.join(log_path, "self_test.err") + cmdfileout = os.path.join(log_path, "self_test.out.log") + cmdfileerr = os.path.join(log_path, "self_test.err.log") + environ['D_LOG_FILE'] = os.path.join(log_path, 'self_test_cart.log') procrtn = -1 try: with open(cmdfileout, mode='w') as outfile, \ @@ -884,7 +886,7 @@ def test_self_test(self): outfile.write("{!s}\n Command: {!s} \n{!s}\n".format( ("=" * 40), (" ".join(cmd)), ("=" * 40))) outfile.flush() - procrtn = subprocess.call(cmd, timeout=180, env=environ, + procrtn = subprocess.call(cmd, timeout=5 * 60, env=environ, stdout=outfile, stderr=errfile) except (FileNotFoundError) as e: self.logger.info("Testnss: %s", \ @@ -892,6 +894,12 @@ def test_self_test(self): except (IOError) as e: self.logger.info("Testnss: Error opening the log files: %s", \ e.errno) + + with open(cmdfileout, "r") as fd: + for line in fd.readlines(): + print(line.strip()) + fd.close() + if procrtn != 0: self.fail("cart self test failed: %s" % procrtn) diff --git a/utils/docker/Dockerfile.centos:7 b/utils/docker/Dockerfile.centos:7 index 0813077c..4ba8eba8 100644 --- a/utils/docker/Dockerfile.centos:7 +++ b/utils/docker/Dockerfile.centos:7 @@ -43,31 +43,57 @@ FROM centos:7 MAINTAINER Johann Lombardi -# Build arguments can be set via -build-arg +# Build arguments can be set via --build-arg # use same UID as host and default value of 1000 if not specified ARG UID=1000 +ARG USE_RPMS=false # for good measure, clean the metadata RUN yum clean metadata # Install basic tools RUN yum -y install epel-release -RUN yum -y install git gcc gcc-c++ make cmake golang libtool scons boost-devel \ - libuuid-devel openssl-devel libevent-devel libtool-ltdl-devel \ - libcmocka libcmocka-devel readline-devel \ - doxygen pandoc flex patch nasm yasm ninja-build meson \ - CUnit-devel libaio-devel python-pep8 lcov \ - python clang-analyzer sg3_utils libiscsi-devel \ - numactl-devel doxygen graphviz pylint python2-pygithub \ - python34 python34-pylint python34-PyYAML.x86_64 PyYAML \ - python2-paramiko python34-paramiko valgrind rsync \ - libyaml-devel file ShellCheck yum-plugin-copr python34-nose \ - python34-pip fuse python34-devel python-devel +RUN yum -y install \ + git gcc gcc-c++ make cmake golang libtool boost-devel \ + libuuid-devel openssl-devel libevent-devel \ + libtool-ltdl-devel libcmocka-devel readline-devel \ + doxygen pandoc flex patch nasm yasm ninja-build meson \ + CUnit-devel libaio-devel clang-analyzer sg3_utils \ + libiscsi-devel numactl-devel graphviz \ + libyaml-devel \ + scons pylint file ShellCheck python-pep8 lcov python \ + python2-pygithub yum-plugin-copr + +# external deps build/repos +# sadly this doesn't work +#RUN yum-config-manager --save --setopt=*_job_daos-stack_job_${ext}_job_master_lastSuccessfulBuild_artifact_artifacts_.gpgcheck=0 +RUN if ! $USE_RPMS; then exit 0; fi; \ +yum -y install hwloc-devel; \ +for ext in openpa libfabric mercury pmix ompi; do \ + yum-config-manager --add-repo=https://build.hpdd.intel.com/job/daos-stack/job/${ext}/job/master/lastSuccessfulBuild/artifact/artifacts/; \ + echo "gpgcheck = False" >> /etc/yum.repos.d/build.hpdd.intel.com_job_daos-stack_job_${ext}_job_master_lastSuccessfulBuild_artifact_artifacts_.repo; \ +done + +ARG OPENPA=unknown +ARG LIBFABRIC=unknown +ARG MERCURY=unknown +ARG PMIX=unknown +ARG OMPI=unknown + +RUN if ! $USE_RPMS; then exit 0; fi; \ +OPENPA=${OPENPA} \ +LIBFABRIC=${LIBFABRIC} \ +MERCURY=${MERCURY} \ +PMIX=${PMIX} \ +OMPI=${OMPI} \ +yum -y install openpa-devel libfabric-devel mercury-devel pmix-devel ompi-devel + RUN yum -y copr enable jhli/ipmctl RUN yum -y copr enable jhli/safeclib -RUN yum -y install libsafec libipmctl-devel +RUN yum -y install libipmctl-devel -RUN pip3.4 install tabulate +RUN yum -y install python34 python34-pylint python34-PyYAML.x86_64 \ + PyYAML python2-paramiko python34-paramiko valgrind rsync # Add CaRT user ENV USER cart @@ -75,6 +101,23 @@ ENV PASSWD cart RUN useradd -u $UID -ms /bin/bash $USER RUN echo "$USER:$PASSWD" | chpasswd +RUN yum -y install python34-nose python34-pip fuse python34-devel python-devel + +RUN pip3.4 install tabulate + +ARG CART_BRANCH=unknown + +RUN if ! $USE_RPMS; then exit 0; fi; \ +for ext in cart; do \ + yum-config-manager --add-repo=https://build.hpdd.intel.com/job/daos-stack/job/${ext}/job/${CART_BRANCH}/lastCompletedBuild/artifact/artifacts/; \ + echo "gpgcheck = False" >> /etc/yum.repos.d/build.hpdd.intel.com_job_daos-stack_job_${ext}_job_${CART_BRANCH}_lastCompletedBuild_artifact_artifacts_.repo; \ +done + +ARG CART_VERSION=unknown + +RUN if ! $USE_RPMS; then exit 0; fi; \ +CART_VERSION=${CART_VERSION} yum -y install cart + # Switch to new user USER $USER WORKDIR /home/$USER