From b6f16a7c033fe87057d9ba44d65bbf24d999228a Mon Sep 17 00:00:00 2001 From: SteBaum Date: Wed, 20 Aug 2025 16:31:51 +0200 Subject: [PATCH 1/3] "feat(tdp-builder): new tdp-builder based on apache hadoop 3.4.1 builder image" This reverts commit 9020cc0e7d814bf67d05609fe25283f1fd28a09d. --- build-env/Dockerfile | 56 +++++++++++++--------------------- build-env/docker-entrypoint.sh | 10 +++--- 2 files changed, 27 insertions(+), 39 deletions(-) diff --git a/build-env/Dockerfile b/build-env/Dockerfile index 1f1329a..09a94cd 100644 --- a/build-env/Dockerfile +++ b/build-env/Dockerfile @@ -18,8 +18,8 @@ # Dockerfile for installing the necessary dependencies for building Hadoop. # See BUILDING.txt. -FROM ubuntu:focal -ARG OS_IDENTIFIER=ubuntu-2004 +FROM ubuntu:bionic +ARG OS_IDENTIFIER=ubuntu-1804 WORKDIR /root @@ -39,8 +39,6 @@ RUN apt-get -q update \ && apt-get -q install -y --no-install-recommends \ ant \ apt-utils \ - autoconf \ - automake \ bats \ build-essential \ bzip2 \ @@ -65,7 +63,7 @@ RUN apt-get -q update \ libsnappy-dev \ libssl-dev \ libtool \ - libzstd-dev \ + libzstd1-dev \ locales \ make \ openjdk-8-jdk \ @@ -83,7 +81,6 @@ RUN apt-get -q update \ sudo \ unzip \ valgrind \ - vim \ wget \ zlib1g-dev \ && apt-get clean \ @@ -94,11 +91,11 @@ ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' LC_ALL='en_US.UTF-8' ENV PYTHONIOENCODING=utf-8 # Install Maven 3.9.9 -RUN mkdir /opt/maven \ +RUN mkdir /opt/maven \ && curl -L https://dlcdn.apache.org/maven/maven-3/3.9.9/binaries/apache-maven-3.9.9-bin.zip -o apache-maven-3.9.9-bin.zip \ && unzip apache-maven-3.9.9-bin.zip \ && mv apache-maven-3.9.9 /opt/maven \ - && rm apache-maven-3.9.9-bin.zip + && rm apache-maven-3.9.9-bin.zip ENV MAVEN_HOME=/opt/maven/apache-maven-3.9.9 ENV PATH=$PATH:$MAVEN_HOME/bin @@ -124,18 +121,16 @@ RUN wget https://cdn.posit.co/r/${OS_IDENTIFIER}/pkgs/r-${R_VERSION}_1_amd64.deb # JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003) ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64 - ####### -# Install Boost 1.86 +# Install Boost 1.72 (1.65 ships with Bionic) ####### - # hadolint ignore=DL3003 -RUN mkdir -p /opt/boost-library \ - && curl -L https://sourceforge.net/projects/boost/files/boost/1.86.0/boost_1_86_0.tar.bz2/download >boost_1_86_0.tar.bz2 \ - && mv boost_1_86_0.tar.bz2 /opt/boost-library \ +RUN mkdir -p /opt/boost-library \ + && curl -L https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download > boost_1_72_0.tar.bz2 \ + && mv boost_1_72_0.tar.bz2 /opt/boost-library \ && cd /opt/boost-library \ - && tar --bzip2 -xf boost_1_86_0.tar.bz2 \ - && cd /opt/boost-library/boost_1_86_0 \ + && tar --bzip2 -xf boost_1_72_0.tar.bz2 \ + && cd /opt/boost-library/boost_1_72_0 \ && ./bootstrap.sh --prefix=/usr/ \ && ./b2 --without-python install \ && cd /root \ @@ -146,22 +141,21 @@ RUN mkdir -p /opt/boost-library \ ####### RUN mkdir -p /opt/spotbugs \ && curl -L -s -S https://github.com/spotbugs/spotbugs/releases/download/4.2.2/spotbugs-4.2.2.tgz \ - -o /opt/spotbugs.tgz \ + -o /opt/spotbugs.tgz \ && tar xzf /opt/spotbugs.tgz --strip-components 1 -C /opt/spotbugs \ && chmod +x /opt/spotbugs/bin/* ENV SPOTBUGS_HOME /opt/spotbugs ###### -# Install Google Protobuf 3.21.12 +# Install Google Protobuf 3.7.1 (3.0.0 ships with Bionic) ###### # hadolint ignore=DL3003 RUN mkdir -p /opt/protobuf-src \ && curl -L -s -S \ - https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.21.12.tar.gz \ - -o /opt/protobuf.tar.gz \ + https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-java-3.7.1.tar.gz \ + -o /opt/protobuf.tar.gz \ && tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src \ - && cd /opt/protobuf-src/ \ - && ./autogen.sh \ + && cd /opt/protobuf-src \ && ./configure --prefix=/opt/protobuf \ && make "-j$(nproc)" \ && make install \ @@ -170,28 +164,22 @@ RUN mkdir -p /opt/protobuf-src \ ENV PROTOBUF_HOME /opt/protobuf ENV PATH "${PATH}:/opt/protobuf/bin" - -# Use venv because of new python system protection -ENV VENV_PATH=/opt/venv -RUN python3 -m venv $VENV_PATH -ENV PATH "$VENV_PATH/bin:$PATH" - #### # Upgrade pip3 #### -RUN python3 -m pip install --upgrade pip setuptools wheel +RUN python3 -m pip install --upgrade pip #### # Install pandas and pyarrow for Spark 3 # venv-pack for jupyterhub venv #### -RUN $VENV_PATH/bin/pip3 install numpy==1.24.4 \ - pandas==2.0.3 \ - pyarrow==14.0.2 \ +RUN pip3 install numpy==1.19.5 \ + pandas==1.0.5 \ + pyarrow==4.0.1 \ venv-pack==0.2.0 # Install pylint and python-dateutil -RUN $VENV_PATH/bin/pip3 install pylint==2.6.0 python-dateutil==2.8.2 +RUN pip3 install pylint==2.6.0 python-dateutil==2.8.1 ### ## Install Yarn 1.12.1 for web UI framework @@ -257,7 +245,7 @@ RUN mkdir -p /opt/gradle \ && curl -L https://services.gradle.org/distributions/gradle-8.1.1-bin.zip -o gradle-8.1.1-bin.zip \ && unzip gradle-8.1.1-bin.zip \ && mv gradle-8.1.1 /opt/gradle \ - && rm gradle-8.1.1-bin.zip + && rm gradle-8.1.1-bin.zip ENV GRADLE_HOME=/opt/gradle/gradle-8.1.1 ENV PATH=$PATH:$GRADLE_HOME/bin diff --git a/build-env/docker-entrypoint.sh b/build-env/docker-entrypoint.sh index db6184f..3df9724 100755 --- a/build-env/docker-entrypoint.sh +++ b/build-env/docker-entrypoint.sh @@ -3,9 +3,9 @@ set -eo pipefail if [[ -n "$BUILDER_UID" ]] && [[ -n "$BUILDER_GID" ]]; then # Create group and user only if they don't exist - [[ ! $(getent group "$BUILDER_GID") ]] && groupadd --gid "$BUILDER_GID" --system builder - if [[ ! $(getent passwd "$BUILDER_UID") ]]; then - useradd --uid "$BUILDER_UID" --system --gid "$BUILDER_GID" --home-dir /home/builder builder + [[ ! $(getent group builder) ]] && groupadd --gid "$BUILDER_GID" --system builder + if [[ ! $(getent passwd builder) ]]; then + useradd --uid "$BUILDER_UID" --system --gid builder --home-dir /home/builder builder # Avoid useradd warning if home dir already exists by making home dir ourselves. # Home dir can exists if mounted via "docker run -v ...:/home/builder/...". mkdir -p /home/builder @@ -15,9 +15,9 @@ if [[ -n "$BUILDER_UID" ]] && [[ -n "$BUILDER_GID" ]]; then # Avoid changing dir if a work dir is specified [[ "$PWD" == "/root" ]] && cd /home/builder if [[ -z "$@" ]]; then - exec gosu $BUILDER_UID /bin/bash + exec gosu builder /bin/bash else - exec gosu $BUILDER_UID "$@" + exec gosu builder "$@" fi fi From 38b16553bc4f443864dfe96c8ec74a35d17eb9e7 Mon Sep 17 00:00:00 2001 From: SteBaum Date: Wed, 20 Aug 2025 16:20:22 +0200 Subject: [PATCH 2/3] fix(tdp_builder): maven 3.9.9 replaced with 3.9.11 since not in official maven repository anymore --- build-env/Dockerfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/build-env/Dockerfile b/build-env/Dockerfile index 09a94cd..697fa67 100644 --- a/build-env/Dockerfile +++ b/build-env/Dockerfile @@ -92,12 +92,12 @@ ENV PYTHONIOENCODING=utf-8 # Install Maven 3.9.9 RUN mkdir /opt/maven \ - && curl -L https://dlcdn.apache.org/maven/maven-3/3.9.9/binaries/apache-maven-3.9.9-bin.zip -o apache-maven-3.9.9-bin.zip \ - && unzip apache-maven-3.9.9-bin.zip \ - && mv apache-maven-3.9.9 /opt/maven \ - && rm apache-maven-3.9.9-bin.zip + && curl -L https://dlcdn.apache.org/maven/maven-3/3.9.11/binaries/apache-maven-3.9.11-bin.zip -o apache-maven-3.9.11-bin.zip \ + && unzip apache-maven-3.9.11-bin.zip \ + && mv apache-maven-3.9.11 /opt/maven \ + && rm apache-maven-3.9.11-bin.zip -ENV MAVEN_HOME=/opt/maven/apache-maven-3.9.9 +ENV MAVEN_HOME=/opt/maven/apache-maven-3.9.11 ENV PATH=$PATH:$MAVEN_HOME/bin ###### From 7b603094fd88546d4e65343f34e7c83779f17193 Mon Sep 17 00:00:00 2001 From: SteBaum Date: Mon, 1 Sep 2025 11:37:04 +0200 Subject: [PATCH 3/3] refactor(tdp-builder): revert to previous docker-entrypoint.sh --- build-env/docker-entrypoint.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/build-env/docker-entrypoint.sh b/build-env/docker-entrypoint.sh index 3df9724..db6184f 100755 --- a/build-env/docker-entrypoint.sh +++ b/build-env/docker-entrypoint.sh @@ -3,9 +3,9 @@ set -eo pipefail if [[ -n "$BUILDER_UID" ]] && [[ -n "$BUILDER_GID" ]]; then # Create group and user only if they don't exist - [[ ! $(getent group builder) ]] && groupadd --gid "$BUILDER_GID" --system builder - if [[ ! $(getent passwd builder) ]]; then - useradd --uid "$BUILDER_UID" --system --gid builder --home-dir /home/builder builder + [[ ! $(getent group "$BUILDER_GID") ]] && groupadd --gid "$BUILDER_GID" --system builder + if [[ ! $(getent passwd "$BUILDER_UID") ]]; then + useradd --uid "$BUILDER_UID" --system --gid "$BUILDER_GID" --home-dir /home/builder builder # Avoid useradd warning if home dir already exists by making home dir ourselves. # Home dir can exists if mounted via "docker run -v ...:/home/builder/...". mkdir -p /home/builder @@ -15,9 +15,9 @@ if [[ -n "$BUILDER_UID" ]] && [[ -n "$BUILDER_GID" ]]; then # Avoid changing dir if a work dir is specified [[ "$PWD" == "/root" ]] && cd /home/builder if [[ -z "$@" ]]; then - exec gosu builder /bin/bash + exec gosu $BUILDER_UID /bin/bash else - exec gosu builder "$@" + exec gosu $BUILDER_UID "$@" fi fi