From 50d09fd36b1cb231a8df2161e79e8e6914ce597c Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Sat, 29 Jan 2022 17:19:01 +0200 Subject: [PATCH 01/36] wip --- Dockerfile | 4 ++-- README.md | 2 +- pyi.Dockerfile | 20 ++++++++++---------- scripts/pyperf_build.sh | 8 -------- 4 files changed, 13 insertions(+), 21 deletions(-) diff --git a/Dockerfile b/Dockerfile index db76e7117..795390dc0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -50,7 +50,7 @@ RUN ./perf_build.sh # pyperf (bcc) FROM ubuntu${PYPERF_BUILDER_UBUNTU} AS bcc-builder-base -RUN apt-get update && apt-get install -y git && if [ $(uname -m) = "aarch64" ]; then exit 0; fi; DEBIAN_FRONTEND=noninteractive apt-get install -y \ +RUN apt-get update && apt-get install -y git && DEBIAN_FRONTEND=noninteractive apt-get install -y \ curl build-essential iperf llvm-9-dev libclang-9-dev cmake python3 flex bison libelf-dev libz-dev liblzma-dev # bcc helpers @@ -66,7 +66,7 @@ RUN ./bcc_helpers_build.sh FROM bcc-builder-base AS bcc-builder COPY ./scripts/libunwind_build.sh . -RUN if [ $(uname -m) = "aarch64" ]; then exit 0; fi; ./libunwind_build.sh +RUN ./libunwind_build.sh WORKDIR /bcc diff --git a/README.md b/README.md index 785b5db2c..42399a2f0 100644 --- a/README.md +++ b/README.md @@ -321,7 +321,7 @@ The runtime stacks are then merged into the data collected by `perf`, substituti | perf (native, Golang, ...) | :heavy_check_mark: | :heavy_check_mark: | | Java (async-profiler) | :heavy_check_mark: | :heavy_check_mark: | | Python (py-spy) | :heavy_check_mark: | :heavy_check_mark: | -| Python (PyPerf eBPF) | :heavy_check_mark: | :x: | +| Python (PyPerf eBPF) | :heavy_check_mark: | :heavy_check_mark: | | Ruby (rbspy) | :heavy_check_mark: | :heavy_check_mark: | | PHP (phpspy) | :heavy_check_mark: | :x: | | NodeJS (perf) | :heavy_check_mark: | :heavy_check_mark: | diff --git a/pyi.Dockerfile b/pyi.Dockerfile index 673e377cb..d8fbe7347 100644 --- a/pyi.Dockerfile +++ b/pyi.Dockerfile @@ -84,7 +84,7 @@ RUN ./burn_build.sh # these are only relevant for modern kernels, so there's no real reason to build them on CentOS 7 anyway. FROM ubuntu${PYPERF_BUILDER_UBUNTU} AS bcc-helpers -RUN if [ $(uname -m) = "aarch64" ]; then exit 0; fi; apt-get update && apt install -y \ +RUN apt-get update && apt install -y \ clang-10 \ libelf-dev \ make \ @@ -107,7 +107,7 @@ FROM centos${GPROFILER_BUILDER} AS build-stage RUN yum install -y git # these are needed to build PyPerf, which we don't build on Aarch64, hence not installing them here. -RUN if [ $(uname -m) = "aarch64" ]; then exit 0; fi; yum install -y \ +RUN yum install -y \ curl \ cmake \ patch \ @@ -119,22 +119,22 @@ RUN if [ $(uname -m) = "aarch64" ]; then exit 0; fi; yum install -y \ ncurses-devel \ elfutils-libelf-devel -RUN if [ $(uname -m) = "aarch64" ]; then exit 0; fi; yum install -y centos-release-scl-rh +RUN yum install -y centos-release-scl-rh # mostly taken from https://github.com/iovisor/bcc/blob/master/INSTALL.md#install-and-compile-llvm -RUN if [ $(uname -m) = "aarch64" ]; then exit 0; fi; yum install -y devtoolset-8 \ - llvm-toolset-7 \ - llvm-toolset-7-llvm-devel \ - llvm-toolset-7-llvm-static \ - llvm-toolset-7-clang-devel \ +RUN yum install -y devtoolset-8 \ + llvm-toolset-7.0 \ + llvm-toolset-7.0-llvm-devel \ + llvm-toolset-7.0-llvm-static \ + llvm-toolset-7.0-clang-devel \ devtoolset-8-elfutils-libelf-devel COPY ./scripts/libunwind_build.sh . -RUN if [ $(uname -m) = "aarch64" ]; then exit 0; fi; ./libunwind_build.sh +RUN ./libunwind_build.sh WORKDIR /bcc COPY ./scripts/pyperf_build.sh . -RUN if [ $(uname -m) != "aarch64" ]; then source scl_source enable devtoolset-8 llvm-toolset-7; fi && source ./pyperf_build.sh +RUN source scl_source enable devtoolset-8 llvm-toolset-7.0 && source ./pyperf_build.sh # gProfiler part diff --git a/scripts/pyperf_build.sh b/scripts/pyperf_build.sh index 0376992fe..7a97a6472 100755 --- a/scripts/pyperf_build.sh +++ b/scripts/pyperf_build.sh @@ -7,14 +7,6 @@ set -e git clone --depth 1 -b v1.2.1 https://github.com/Granulate/bcc.git && cd bcc && git reset --hard 3f1d60180946852aa89e3644b786e4bf8934e7a9 -# (after clone, because we copy the licenses) -# TODO support aarch64 -if [ $(uname -m) != "x86_64" ]; then - mkdir -p /bcc/root/share/bcc/examples/cpp/ - touch /bcc/root/share/bcc/examples/cpp/PyPerf - exit 0 -fi - mkdir build cd build cmake -DPYTHON_CMD=python3 -DINSTALL_CPP_EXAMPLES=y -DCMAKE_INSTALL_PREFIX=/bcc/root .. From 4c66f633fc5d2b742b8ec4051677fa09c845024f Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Tue, 1 Feb 2022 01:22:51 +0200 Subject: [PATCH 02/36] python: Allow PyPerf on aarch64 --- gprofiler/profilers/python.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/gprofiler/profilers/python.py b/gprofiler/profilers/python.py index 20661ef1a..627868e45 100644 --- a/gprofiler/profilers/python.py +++ b/gprofiler/profilers/python.py @@ -26,7 +26,6 @@ from gprofiler.log import get_logger_adapter from gprofiler.merge import parse_many_collapsed, parse_one_collapsed_file from gprofiler.metadata.py_module_version import get_modules_versions -from gprofiler.metadata.system_metadata import get_arch from gprofiler.profilers.profiler_base import ProcessProfilerBase, ProfilerBase, ProfilerInterface from gprofiler.profilers.registry import ProfilerArgument, register_profiler from gprofiler.utils import ( @@ -395,9 +394,6 @@ def stop(self): # py-spy is like pyspy, it's confusing and I mix between them possible_modes=["auto", "pyperf", "pyspy", "py-spy", "disabled"], default_mode="auto", - # we build pyspy for both, pyperf only for x86_64. - # TODO: this inconsistency shows that py-spy and pyperf should have different Profiler classes, - # we should split them in the future. supported_archs=["x86_64", "aarch64"], profiler_mode_argument_help="Select the Python profiling mode: auto (try PyPerf, resort to py-spy if it fails), " "pyspy (always use py-spy), pyperf (always use PyPerf, and avoid py-spy even if it fails)" @@ -443,11 +439,6 @@ def __init__( assert python_mode in ("auto", "pyperf", "pyspy"), f"unexpected mode: {python_mode}" - if get_arch() != "x86_64": - if python_mode == "pyperf": - logger.warning("PyPerf is supported only on x86_64, falling back to py-spy") - python_mode = "pyspy" - if python_mode in ("auto", "pyperf"): self._ebpf_profiler = self._create_ebpf_profiler( frequency, duration, stop_event, storage_dir, python_add_versions, python_pyperf_user_stacks_pages From c5937a6e1181ad91f195a0d3ac8fa66a89a4f0db Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Tue, 1 Feb 2022 01:23:06 +0200 Subject: [PATCH 03/36] wip update bpf helpers to aarch64 revisions --- scripts/bcc_helpers_build.sh | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/scripts/bcc_helpers_build.sh b/scripts/bcc_helpers_build.sh index 096d57d15..9ca892ce6 100755 --- a/scripts/bcc_helpers_build.sh +++ b/scripts/bcc_helpers_build.sh @@ -5,14 +5,6 @@ # set -euo pipefail -# TODO support aarch64, after we support it in PyPerf -if [ $(uname -m) != "x86_64" ]; then - mkdir -p /bpf_get_fs_offset /bpf_get_stack_offset - touch /bpf_get_fs_offset/get_fs_offset - touch /bpf_get_stack_offset/get_stack_offset - exit 0 -fi - LLVM_STRIP=llvm-strip if ! command -v "$LLVM_STRIP" > /dev/null 2>&1 ; then LLVM_STRIP=llvm-strip-10 @@ -20,10 +12,10 @@ fi LIBBPF_MAKE_FLAGS="BPFTOOL=/bpftool CLANG=clang-10 LLVM_STRIP=$LLVM_STRIP CFLAGS=-static" -cd / && git clone -b v0.0.2 --depth=1 --recurse-submodules https://github.com/Jongy/bpf_get_fs_offset.git -cd /bpf_get_fs_offset && git reset --hard 8326d39cf44845d4b643ed4267994afca8ccecb3 +cd / && git clone -b aarch64 --depth=1 --recurse-submodules https://github.com/Jongy/bpf_get_fs_offset.git +cd /bpf_get_fs_offset && git reset --hard 094e93f979308d46dffb8d4ea88823f68d53ba85 cd /bpf_get_fs_offset && make $LIBBPF_MAKE_FLAGS -cd / && git clone -b v0.0.3 --depth=1 --recurse-submodules https://github.com/Jongy/bpf_get_stack_offset.git -cd /bpf_get_stack_offset && git reset --hard 54b70ee65708cc8d3d7817277e82376d95205356 +cd / && git clone -b aarch64 --depth=1 --recurse-submodules https://github.com/Jongy/bpf_get_stack_offset.git +cd /bpf_get_stack_offset && git reset --hard d8b77ce6da674c38ad0bb856686fde1e63ad0814 cd /bpf_get_stack_offset && make $LIBBPF_MAKE_FLAGS From 76eba37fcb9269076a7e4c792bb9e8954afb0a56 Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Tue, 1 Feb 2022 01:24:34 +0200 Subject: [PATCH 04/36] wip --- scripts/pyperf_build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/pyperf_build.sh b/scripts/pyperf_build.sh index 7a97a6472..ee362cc15 100755 --- a/scripts/pyperf_build.sh +++ b/scripts/pyperf_build.sh @@ -5,7 +5,7 @@ # set -e -git clone --depth 1 -b v1.2.1 https://github.com/Granulate/bcc.git && cd bcc && git reset --hard 3f1d60180946852aa89e3644b786e4bf8934e7a9 +git clone --depth 1 -b aarch64 https://github.com/Granulate/bcc.git && cd bcc && git reset --hard 1d502e0ffb93bbfec6d77dddeb7668f0a90e6810 mkdir build cd build From f4151dcb77dec6ed7b69bdfdcc8ac7330db68f7e Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Tue, 1 Feb 2022 01:55:01 +0200 Subject: [PATCH 05/36] more aarch64 fixes --- pyi.Dockerfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pyi.Dockerfile b/pyi.Dockerfile index d8fbe7347..e6c5d30e6 100644 --- a/pyi.Dockerfile +++ b/pyi.Dockerfile @@ -106,7 +106,6 @@ FROM centos${GPROFILER_BUILDER} AS build-stage RUN yum install -y git -# these are needed to build PyPerf, which we don't build on Aarch64, hence not installing them here. RUN yum install -y \ curl \ cmake \ @@ -206,8 +205,7 @@ COPY ./scripts/list_needed_libs.sh ./scripts/list_needed_libs.sh # we use list_needed_libs.sh to list the dynamic dependencies of *all* of our resources, # and make staticx pack them as well. # using scl here to get the proper LD_LIBRARY_PATH set -# TODO: use staticx for aarch64 as well; currently it doesn't generate correct binaries when run over Docker emulation. -RUN if [ $(uname -m) != "aarch64" ]; then source scl_source enable devtoolset-8 llvm-toolset-7 && libs=$(./scripts/list_needed_libs.sh) && staticx $libs dist/gprofiler dist/gprofiler; fi +RUN then source scl_source enable devtoolset-8 llvm-toolset-7 && libs=$(./scripts/list_needed_libs.sh) && staticx $libs dist/gprofiler dist/gprofiler FROM scratch AS export-stage From e666923a3c8f32bfb46675bc7e012fbeb08daaa8 Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Tue, 1 Feb 2022 01:55:40 +0200 Subject: [PATCH 06/36] . --- pyi.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyi.Dockerfile b/pyi.Dockerfile index e6c5d30e6..e37f18e8d 100644 --- a/pyi.Dockerfile +++ b/pyi.Dockerfile @@ -205,7 +205,7 @@ COPY ./scripts/list_needed_libs.sh ./scripts/list_needed_libs.sh # we use list_needed_libs.sh to list the dynamic dependencies of *all* of our resources, # and make staticx pack them as well. # using scl here to get the proper LD_LIBRARY_PATH set -RUN then source scl_source enable devtoolset-8 llvm-toolset-7 && libs=$(./scripts/list_needed_libs.sh) && staticx $libs dist/gprofiler dist/gprofiler +RUN source scl_source enable devtoolset-8 llvm-toolset-7 && libs=$(./scripts/list_needed_libs.sh) && staticx $libs dist/gprofiler dist/gprofiler FROM scratch AS export-stage From d5cdcd857be37f329703d841a51f9f63f433a89e Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Tue, 1 Feb 2022 01:56:29 +0200 Subject: [PATCH 07/36] . --- pyi.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyi.Dockerfile b/pyi.Dockerfile index e37f18e8d..849390e77 100644 --- a/pyi.Dockerfile +++ b/pyi.Dockerfile @@ -205,7 +205,7 @@ COPY ./scripts/list_needed_libs.sh ./scripts/list_needed_libs.sh # we use list_needed_libs.sh to list the dynamic dependencies of *all* of our resources, # and make staticx pack them as well. # using scl here to get the proper LD_LIBRARY_PATH set -RUN source scl_source enable devtoolset-8 llvm-toolset-7 && libs=$(./scripts/list_needed_libs.sh) && staticx $libs dist/gprofiler dist/gprofiler +RUN source scl_source enable devtoolset-8 llvm-toolset-7.0 && libs=$(./scripts/list_needed_libs.sh) && staticx $libs dist/gprofiler dist/gprofiler FROM scratch AS export-stage From 0c858d509e91f7f639723ac917554d17e707aaf9 Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Tue, 1 Feb 2022 02:01:01 +0200 Subject: [PATCH 08/36] Add -DENABLE_LLVM_SHARED=1 --- scripts/pyperf_build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/pyperf_build.sh b/scripts/pyperf_build.sh index ee362cc15..f3daae1ef 100755 --- a/scripts/pyperf_build.sh +++ b/scripts/pyperf_build.sh @@ -9,5 +9,5 @@ git clone --depth 1 -b aarch64 https://github.com/Granulate/bcc.git && cd bcc && mkdir build cd build -cmake -DPYTHON_CMD=python3 -DINSTALL_CPP_EXAMPLES=y -DCMAKE_INSTALL_PREFIX=/bcc/root .. +cmake -DPYTHON_CMD=python3 -DINSTALL_CPP_EXAMPLES=y -DCMAKE_INSTALL_PREFIX=/bcc/root -DENABLE_LLVM_SHARED=1 .. make -C examples/cpp/pyperf -j -l VERBOSE=1 install From de1ccf47dbd6b459f2e37fab7f66f6fee55404ca Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Tue, 1 Feb 2022 02:08:39 +0200 Subject: [PATCH 09/36] upgrade bcc --- scripts/pyperf_build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/pyperf_build.sh b/scripts/pyperf_build.sh index f3daae1ef..a5ab6baa9 100755 --- a/scripts/pyperf_build.sh +++ b/scripts/pyperf_build.sh @@ -5,7 +5,7 @@ # set -e -git clone --depth 1 -b aarch64 https://github.com/Granulate/bcc.git && cd bcc && git reset --hard 1d502e0ffb93bbfec6d77dddeb7668f0a90e6810 +git clone --depth 1 -b aarch64 https://github.com/Granulate/bcc.git && cd bcc && git reset --hard e9632c3e9d7a1ad2d58e1abb6cf86ccfdaa81549 mkdir build cd build From 3e6c9b78af266be340c5f6f9d1ec3884d53ff31a Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Tue, 1 Feb 2022 02:15:27 +0200 Subject: [PATCH 10/36] . --- scripts/pyperf_build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/pyperf_build.sh b/scripts/pyperf_build.sh index a5ab6baa9..71d72a3a8 100755 --- a/scripts/pyperf_build.sh +++ b/scripts/pyperf_build.sh @@ -5,7 +5,7 @@ # set -e -git clone --depth 1 -b aarch64 https://github.com/Granulate/bcc.git && cd bcc && git reset --hard e9632c3e9d7a1ad2d58e1abb6cf86ccfdaa81549 +git clone --depth 1 -b aarch64 https://github.com/Granulate/bcc.git && cd bcc && git reset --hard cf3405b7f35f910b5495f4804cfe65630a2688b6 mkdir build cd build From ff06bb1c69b0739d04a68edaa3b9e944a49f8d52 Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Tue, 1 Feb 2022 02:23:48 +0200 Subject: [PATCH 11/36] fix llvm versioning --- pyi.Dockerfile | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/pyi.Dockerfile b/pyi.Dockerfile index 849390e77..f9b13b85b 100644 --- a/pyi.Dockerfile +++ b/pyi.Dockerfile @@ -120,11 +120,12 @@ RUN yum install -y \ RUN yum install -y centos-release-scl-rh # mostly taken from https://github.com/iovisor/bcc/blob/master/INSTALL.md#install-and-compile-llvm -RUN yum install -y devtoolset-8 \ - llvm-toolset-7.0 \ - llvm-toolset-7.0-llvm-devel \ - llvm-toolset-7.0-llvm-static \ - llvm-toolset-7.0-clang-devel \ +# on x86_64, the package is named llvm-toolset-7. on aarch64, it is named llvm-toolset-7.0... +RUN if [ $(uname -m) = "aarch64" ]; then v="7.0"; else v="7"; fi; yum install -y devtoolset-8 \ + llvm-toolset-$v \ + llvm-toolset-$v-llvm-devel \ + llvm-toolset-$v-llvm-static \ + llvm-toolset-$v-clang-devel \ devtoolset-8-elfutils-libelf-devel COPY ./scripts/libunwind_build.sh . @@ -205,7 +206,7 @@ COPY ./scripts/list_needed_libs.sh ./scripts/list_needed_libs.sh # we use list_needed_libs.sh to list the dynamic dependencies of *all* of our resources, # and make staticx pack them as well. # using scl here to get the proper LD_LIBRARY_PATH set -RUN source scl_source enable devtoolset-8 llvm-toolset-7.0 && libs=$(./scripts/list_needed_libs.sh) && staticx $libs dist/gprofiler dist/gprofiler +RUN if [ $(uname -m) = "aarch64" ]; then v="7.0"; else v="7"; fi; source scl_source enable devtoolset-8 llvm-toolset-$v && libs=$(./scripts/list_needed_libs.sh) && staticx $libs dist/gprofiler dist/gprofiler FROM scratch AS export-stage From 2f9d8803c7b33bb8937342d4ef9ff750da51281e Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Tue, 1 Feb 2022 02:35:59 +0200 Subject: [PATCH 12/36] wip --- scripts/pyperf_build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/pyperf_build.sh b/scripts/pyperf_build.sh index 71d72a3a8..780d3b1ef 100755 --- a/scripts/pyperf_build.sh +++ b/scripts/pyperf_build.sh @@ -5,7 +5,7 @@ # set -e -git clone --depth 1 -b aarch64 https://github.com/Granulate/bcc.git && cd bcc && git reset --hard cf3405b7f35f910b5495f4804cfe65630a2688b6 +git clone --depth 1 -b aarch64 https://github.com/Granulate/bcc.git && cd bcc && git reset --hard 918fc3138beac21262f5f3b96f92b37b97837bf0 mkdir build cd build From 9de406b4d17f6f6f1b6d71ec47c7aae9309d9077 Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Tue, 1 Feb 2022 02:49:54 +0200 Subject: [PATCH 13/36] wip --- scripts/pyperf_build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/pyperf_build.sh b/scripts/pyperf_build.sh index 780d3b1ef..16443aba4 100755 --- a/scripts/pyperf_build.sh +++ b/scripts/pyperf_build.sh @@ -5,7 +5,7 @@ # set -e -git clone --depth 1 -b aarch64 https://github.com/Granulate/bcc.git && cd bcc && git reset --hard 918fc3138beac21262f5f3b96f92b37b97837bf0 +git clone --depth 1 -b aarch64 https://github.com/Granulate/bcc.git && cd bcc && git reset --hard 95fac2db6e51d3088e076b38eed1d072c8d2d43d mkdir build cd build From 99b12f4429b7984ea42dbb9d63bfd7e7bc897702 Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Tue, 1 Feb 2022 02:52:59 +0200 Subject: [PATCH 14/36] wip --- scripts/pyperf_build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/pyperf_build.sh b/scripts/pyperf_build.sh index 16443aba4..b9f698681 100755 --- a/scripts/pyperf_build.sh +++ b/scripts/pyperf_build.sh @@ -5,7 +5,7 @@ # set -e -git clone --depth 1 -b aarch64 https://github.com/Granulate/bcc.git && cd bcc && git reset --hard 95fac2db6e51d3088e076b38eed1d072c8d2d43d +git clone --depth 1 -b aarch64 https://github.com/Granulate/bcc.git && cd bcc && git reset --hard e4de0ff3ff8174377ea08ef049dfec60418608da mkdir build cd build From bcf3274fd5704be326df956e8dd411d3239ab576 Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Tue, 1 Feb 2022 02:55:33 +0200 Subject: [PATCH 15/36] wip --- scripts/pyperf_build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/pyperf_build.sh b/scripts/pyperf_build.sh index b9f698681..92981091e 100755 --- a/scripts/pyperf_build.sh +++ b/scripts/pyperf_build.sh @@ -5,7 +5,7 @@ # set -e -git clone --depth 1 -b aarch64 https://github.com/Granulate/bcc.git && cd bcc && git reset --hard e4de0ff3ff8174377ea08ef049dfec60418608da +git clone --depth 1 -b aarch64 https://github.com/Granulate/bcc.git && cd bcc && git reset --hard 8347ece735029404c743806a2f1325bc35352673 mkdir build cd build From 1ae29547610ca7de0085cb526cfaf00d72e2a010 Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Tue, 1 Feb 2022 03:06:11 +0200 Subject: [PATCH 16/36] wip --- scripts/pyperf_build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/pyperf_build.sh b/scripts/pyperf_build.sh index 92981091e..7e5ce12e7 100755 --- a/scripts/pyperf_build.sh +++ b/scripts/pyperf_build.sh @@ -5,7 +5,7 @@ # set -e -git clone --depth 1 -b aarch64 https://github.com/Granulate/bcc.git && cd bcc && git reset --hard 8347ece735029404c743806a2f1325bc35352673 +git clone --depth 1 -b aarch64 https://github.com/Granulate/bcc.git && cd bcc && git reset --hard fd3cd7510161a2b7b02a5aff234e4a71b8e8486c mkdir build cd build From e5b3f32597f5e82956144976d56712b4be55a733 Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Tue, 1 Feb 2022 03:10:01 +0200 Subject: [PATCH 17/36] wip --- scripts/pyperf_build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/pyperf_build.sh b/scripts/pyperf_build.sh index 7e5ce12e7..5a8716e2f 100755 --- a/scripts/pyperf_build.sh +++ b/scripts/pyperf_build.sh @@ -5,7 +5,7 @@ # set -e -git clone --depth 1 -b aarch64 https://github.com/Granulate/bcc.git && cd bcc && git reset --hard fd3cd7510161a2b7b02a5aff234e4a71b8e8486c +git clone --depth 1 -b aarch64 https://github.com/Granulate/bcc.git && cd bcc && git reset --hard b33dd19493fc7fc3c5c76935f17b2a67c7b30c9c mkdir build cd build From a42596c241320a2204e7d511f4a963e44b1fed6d Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Tue, 1 Feb 2022 03:25:55 +0200 Subject: [PATCH 18/36] wip --- scripts/pyperf_build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/pyperf_build.sh b/scripts/pyperf_build.sh index 5a8716e2f..d2abce5a6 100755 --- a/scripts/pyperf_build.sh +++ b/scripts/pyperf_build.sh @@ -5,7 +5,7 @@ # set -e -git clone --depth 1 -b aarch64 https://github.com/Granulate/bcc.git && cd bcc && git reset --hard b33dd19493fc7fc3c5c76935f17b2a67c7b30c9c +git clone --depth 1 -b aarch64 https://github.com/Granulate/bcc.git && cd bcc && git reset --hard 91a4accb4c114a6b2185c1622b92437f16a09acc mkdir build cd build From d86dfb0f247d4863f76864cbf92a607a3cf0acf3 Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Thu, 3 Feb 2022 01:10:46 +0200 Subject: [PATCH 19/36] pyperf verbose --- gprofiler/profilers/python.py | 1 + 1 file changed, 1 insertion(+) diff --git a/gprofiler/profilers/python.py b/gprofiler/profilers/python.py index 627868e45..2aa983e55 100644 --- a/gprofiler/profilers/python.py +++ b/gprofiler/profilers/python.py @@ -310,6 +310,7 @@ def start(self): logger.info("Starting profiling of Python processes with PyPerf") cmd = [ resource_path(self.PYPERF_RESOURCE), + "-v", "--output", str(self.output_path), "-F", From a0f8d6acc0aa79fd970c9feae949dc3eeed8c557 Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Thu, 3 Feb 2022 01:12:42 +0200 Subject: [PATCH 20/36] wip --- gprofiler/profilers/python.py | 1 + 1 file changed, 1 insertion(+) diff --git a/gprofiler/profilers/python.py b/gprofiler/profilers/python.py index 2aa983e55..26a11974a 100644 --- a/gprofiler/profilers/python.py +++ b/gprofiler/profilers/python.py @@ -311,6 +311,7 @@ def start(self): cmd = [ resource_path(self.PYPERF_RESOURCE), "-v", + "999", "--output", str(self.output_path), "-F", From 69dbff090c1e6675a17adc568ae9977accfcf377 Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Thu, 3 Feb 2022 01:25:38 +0200 Subject: [PATCH 21/36] wip --- scripts/pyperf_build.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/pyperf_build.sh b/scripts/pyperf_build.sh index d2abce5a6..9b15b811f 100755 --- a/scripts/pyperf_build.sh +++ b/scripts/pyperf_build.sh @@ -5,9 +5,10 @@ # set -e -git clone --depth 1 -b aarch64 https://github.com/Granulate/bcc.git && cd bcc && git reset --hard 91a4accb4c114a6b2185c1622b92437f16a09acc +git clone --depth 1 -b aarch64 https://github.com/Granulate/bcc.git && cd bcc && git reset --hard aba6bdc0dcf089128b5d74f4b30ee0d86b56567b mkdir build cd build +# TODO -DENABLE_LLVM_SHARED only for aarch exe cmake -DPYTHON_CMD=python3 -DINSTALL_CPP_EXAMPLES=y -DCMAKE_INSTALL_PREFIX=/bcc/root -DENABLE_LLVM_SHARED=1 .. make -C examples/cpp/pyperf -j -l VERBOSE=1 install From c0677d2dd6f0c80f825a01a13fa31a0cc824eee6 Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Thu, 3 Feb 2022 01:34:44 +0200 Subject: [PATCH 22/36] wip --- Dockerfile | 2 +- pyi.Dockerfile | 2 +- scripts/pyperf_build.sh | 13 ++++++++++--- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 795390dc0..ff8fd3bc8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -71,7 +71,7 @@ RUN ./libunwind_build.sh WORKDIR /bcc COPY ./scripts/pyperf_build.sh . -RUN ./pyperf_build.sh +RUN ./pyperf_build.sh container # phpspy FROM ubuntu${PHPSPY_BUILDER_UBUNTU} AS phpspy-builder diff --git a/pyi.Dockerfile b/pyi.Dockerfile index f9b13b85b..159b967c3 100644 --- a/pyi.Dockerfile +++ b/pyi.Dockerfile @@ -134,7 +134,7 @@ RUN ./libunwind_build.sh WORKDIR /bcc COPY ./scripts/pyperf_build.sh . -RUN source scl_source enable devtoolset-8 llvm-toolset-7.0 && source ./pyperf_build.sh +RUN source scl_source enable devtoolset-8 llvm-toolset-7.0 && source ./pyperf_build.sh exe # gProfiler part diff --git a/scripts/pyperf_build.sh b/scripts/pyperf_build.sh index 9b15b811f..b1dd98b70 100755 --- a/scripts/pyperf_build.sh +++ b/scripts/pyperf_build.sh @@ -5,10 +5,17 @@ # set -e -git clone --depth 1 -b aarch64 https://github.com/Granulate/bcc.git && cd bcc && git reset --hard aba6bdc0dcf089128b5d74f4b30ee0d86b56567b +git clone --depth 1 -b aarch64 https://github.com/Granulate/bcc.git && cd bcc && git reset --hard bb466dfa4ab315e1dd457b708090cc84552e01e7 mkdir build cd build -# TODO -DENABLE_LLVM_SHARED only for aarch exe -cmake -DPYTHON_CMD=python3 -DINSTALL_CPP_EXAMPLES=y -DCMAKE_INSTALL_PREFIX=/bcc/root -DENABLE_LLVM_SHARED=1 .. + +SHARED_ARG="" +# need in aarch64 as mentioned here: https://github.com/iovisor/bcc/issues/3333#issuecomment-803432248 +# container mdoe doesn't want it - we don't have the libs bundled. +# exe mode in x86_64 works fine so I don't change it. +if [ $(uname -m) = "aarch64" ] && [ "$1" == "exe" ]; then + SHARED_ARG=" -DENABLE_LLVM_SHARED=1" +fi +cmake -DPYTHON_CMD=python3 -DINSTALL_CPP_EXAMPLES=y -DCMAKE_INSTALL_PREFIX=/bcc/root $SHARED_ARG .. make -C examples/cpp/pyperf -j -l VERBOSE=1 install From 6db367d886122af31b6eca8f6f731ce5aafdc60d Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Thu, 3 Feb 2022 08:55:35 +0200 Subject: [PATCH 23/36] wip --- scripts/pyperf_build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/pyperf_build.sh b/scripts/pyperf_build.sh index b1dd98b70..74377b94e 100755 --- a/scripts/pyperf_build.sh +++ b/scripts/pyperf_build.sh @@ -5,7 +5,7 @@ # set -e -git clone --depth 1 -b aarch64 https://github.com/Granulate/bcc.git && cd bcc && git reset --hard bb466dfa4ab315e1dd457b708090cc84552e01e7 +git clone --depth 1 -b aarch64 https://github.com/Granulate/bcc.git && cd bcc && git reset --hard c7461900706a6cc710c8ba5aa47a9282c5ce9cfb mkdir build cd build From af427248804b46d953a893b71780e3fb1af1a74d Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Fri, 4 Feb 2022 02:34:46 +0200 Subject: [PATCH 24/36] wip --- scripts/pyperf_build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/pyperf_build.sh b/scripts/pyperf_build.sh index 74377b94e..3ad239b25 100755 --- a/scripts/pyperf_build.sh +++ b/scripts/pyperf_build.sh @@ -5,7 +5,7 @@ # set -e -git clone --depth 1 -b aarch64 https://github.com/Granulate/bcc.git && cd bcc && git reset --hard c7461900706a6cc710c8ba5aa47a9282c5ce9cfb +git clone --depth 1 -b aarch64 https://github.com/Granulate/bcc.git && cd bcc && git reset --hard 994959b6dd4f9e4f8fc8212110a1dd7d5be9b33d mkdir build cd build From 118f4118453f4ddddebd90a6529c760fa256bea7 Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Fri, 4 Feb 2022 02:42:28 +0200 Subject: [PATCH 25/36] wip --- scripts/pyperf_build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/pyperf_build.sh b/scripts/pyperf_build.sh index 3ad239b25..a2445a8c5 100755 --- a/scripts/pyperf_build.sh +++ b/scripts/pyperf_build.sh @@ -5,7 +5,7 @@ # set -e -git clone --depth 1 -b aarch64 https://github.com/Granulate/bcc.git && cd bcc && git reset --hard 994959b6dd4f9e4f8fc8212110a1dd7d5be9b33d +git clone --depth 1 -b aarch64 https://github.com/Granulate/bcc.git && cd bcc && git reset --hard e7f0b07f73221aedbbf0d63a745c51acdc7f4b13 mkdir build cd build From 9a0e53bc452464b542356f91929ddd0b3c30497a Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Mon, 17 Oct 2022 01:18:15 +0300 Subject: [PATCH 26/36] remove x86_64 reference --- pyi.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyi.Dockerfile b/pyi.Dockerfile index 058961ee8..915edd44f 100644 --- a/pyi.Dockerfile +++ b/pyi.Dockerfile @@ -162,7 +162,7 @@ RUN yum install -y \ python3 \ flex \ bison \ - zlib-devel.x86_64 \ + zlib-devel \ xz-devel \ ncurses-devel \ elfutils-libelf-devel && \ From 77929205e443e3fe87580e2c790ff666af34cc55 Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Wed, 30 Nov 2022 01:43:23 +0200 Subject: [PATCH 27/36] builds again --- Dockerfile | 2 +- scripts/pyperf_build.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 577536622..605aa056e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -98,7 +98,7 @@ RUN apt-get update && \ bison \ libelf-dev \ libz-dev \ - liblzma-dev; \ + liblzma-dev; # bcc helpers FROM bcc-builder-base AS bcc-helpers diff --git a/scripts/pyperf_build.sh b/scripts/pyperf_build.sh index 08b748179..ac6311b8b 100755 --- a/scripts/pyperf_build.sh +++ b/scripts/pyperf_build.sh @@ -5,7 +5,7 @@ # set -euo pipefail -git clone --depth 1 -b aarch64 https://github.com/Granulate/bcc.git && cd bcc && git reset --hard e7f0b07f73221aedbbf0d63a745c51acdc7f4b13 +git clone --depth 1 -b aarch64 https://github.com/Granulate/bcc.git && cd bcc && git reset --hard b2c69412b3abaaf2bef02ff95459f4572574431d mkdir build cd build From 2adb7eb152944497d0fc6e1848829a63c90e3894 Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Wed, 30 Nov 2022 01:45:37 +0200 Subject: [PATCH 28/36] fixes --- pyi.Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pyi.Dockerfile b/pyi.Dockerfile index 9b906de3b..21eaa5f3e 100644 --- a/pyi.Dockerfile +++ b/pyi.Dockerfile @@ -127,7 +127,7 @@ USER 1001 FROM ubuntu${PYPERF_BUILDER_UBUNTU} AS bcc-helpers WORKDIR /tmp -RUN apt-get update && apt install -y --no-install-recommends \ +RUN apt-get update && apt-get install -y --no-install-recommends \ clang-10 \ libelf-dev \ make \ @@ -334,7 +334,6 @@ COPY ./scripts/list_needed_libs.sh ./scripts/list_needed_libs.sh RUN set -e; \ if [ $(uname -m) = "aarch64" ]; then v="7.0"; else v="7"; fi; \ source scl_source enable devtoolset-8 "llvm-toolset-$v" ; \ - fi && \ LIBS=$(./scripts/list_needed_libs.sh) && \ staticx $LIBS dist/gprofiler dist/gprofiler From 2e036d74215d15a62e33b09496dcc660a2d6a13c Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Wed, 30 Nov 2022 01:48:00 +0200 Subject: [PATCH 29/36] update rev --- scripts/pyperf_build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/pyperf_build.sh b/scripts/pyperf_build.sh index ac6311b8b..692c0cc71 100755 --- a/scripts/pyperf_build.sh +++ b/scripts/pyperf_build.sh @@ -5,7 +5,7 @@ # set -euo pipefail -git clone --depth 1 -b aarch64 https://github.com/Granulate/bcc.git && cd bcc && git reset --hard b2c69412b3abaaf2bef02ff95459f4572574431d +git clone --depth 1 -b aarch64 https://github.com/Granulate/bcc.git && cd bcc && git reset --hard 747cd10fc32514a221d89547e4ca8b1d2b10843d mkdir build cd build From 29c209d2ac4ceca029c12fbe1bdac7e65ee5a3cb Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Wed, 30 Nov 2022 01:54:23 +0200 Subject: [PATCH 30/36] fix --- pyi.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyi.Dockerfile b/pyi.Dockerfile index 21eaa5f3e..c4b8af600 100644 --- a/pyi.Dockerfile +++ b/pyi.Dockerfile @@ -333,7 +333,7 @@ COPY ./scripts/list_needed_libs.sh ./scripts/list_needed_libs.sh # hadolint ignore=SC2046,SC2086 RUN set -e; \ if [ $(uname -m) = "aarch64" ]; then v="7.0"; else v="7"; fi; \ - source scl_source enable devtoolset-8 "llvm-toolset-$v" ; \ + source scl_source enable devtoolset-8 "llvm-toolset-$v" && \ LIBS=$(./scripts/list_needed_libs.sh) && \ staticx $LIBS dist/gprofiler dist/gprofiler From 890c56abbfe7aee7a3e45bb53022273051dfbb79 Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Wed, 30 Nov 2022 02:01:19 +0200 Subject: [PATCH 31/36] fix shellcheck --- scripts/pyperf_build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/pyperf_build.sh b/scripts/pyperf_build.sh index 692c0cc71..910ed3dad 100755 --- a/scripts/pyperf_build.sh +++ b/scripts/pyperf_build.sh @@ -14,8 +14,8 @@ SHARED_ARG="" # need in aarch64 as mentioned here: https://github.com/iovisor/bcc/issues/3333#issuecomment-803432248 # container mdoe doesn't want it - we don't have the libs bundled. # exe mode in x86_64 works fine so I don't change it. -if [ $(uname -m) = "aarch64" ] && [ "$1" == "exe" ]; then +if [ "$(uname -m)" = "aarch64" ] && [ "$1" == "exe" ]; then SHARED_ARG=" -DENABLE_LLVM_SHARED=1" fi -cmake -DPYTHON_CMD=python3 -DINSTALL_CPP_EXAMPLES=y -DCMAKE_INSTALL_PREFIX=/bcc/root $SHARED_ARG .. +cmake -DPYTHON_CMD=python3 -DINSTALL_CPP_EXAMPLES=y -DCMAKE_INSTALL_PREFIX=/bcc/root "$SHARED_ARG" .. make -C examples/cpp/pyperf -j -l VERBOSE=1 install From 6ae66d1ef4d4b159a79a099ac8d2c9332b584e59 Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Wed, 30 Nov 2022 02:43:16 +0200 Subject: [PATCH 32/36] test update granulate-utils --- granulate-utils | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/granulate-utils b/granulate-utils index cb9f9822c..2d9b3c1ee 160000 --- a/granulate-utils +++ b/granulate-utils @@ -1 +1 @@ -Subproject commit cb9f9822cf53d662753ae913a35af8c320a07024 +Subproject commit 2d9b3c1ee87ee860839990565effeb4f01c61693 From c0d3d995d43a84d51d1832fe60d7937204eaf248 Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Sat, 17 Dec 2022 02:21:28 +0200 Subject: [PATCH 33/36] update build --- scripts/pyperf_build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/pyperf_build.sh b/scripts/pyperf_build.sh index 910ed3dad..6d76adf9b 100755 --- a/scripts/pyperf_build.sh +++ b/scripts/pyperf_build.sh @@ -5,7 +5,7 @@ # set -euo pipefail -git clone --depth 1 -b aarch64 https://github.com/Granulate/bcc.git && cd bcc && git reset --hard 747cd10fc32514a221d89547e4ca8b1d2b10843d +git clone --depth 1 -b aarch64 https://github.com/Granulate/bcc.git && cd bcc && git reset --hard 704c7f4334b3562bc4bb8e93e139ad0b211005fe mkdir build cd build From ebbe8300fd23d119063b26e67384d562f9c1ddad Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Sat, 17 Dec 2022 02:25:28 +0200 Subject: [PATCH 34/36] update --- scripts/pyperf_build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/pyperf_build.sh b/scripts/pyperf_build.sh index 6d76adf9b..46a0218b5 100755 --- a/scripts/pyperf_build.sh +++ b/scripts/pyperf_build.sh @@ -5,7 +5,7 @@ # set -euo pipefail -git clone --depth 1 -b aarch64 https://github.com/Granulate/bcc.git && cd bcc && git reset --hard 704c7f4334b3562bc4bb8e93e139ad0b211005fe +git clone --depth 1 -b aarch64 https://github.com/Granulate/bcc.git && cd bcc && git reset --hard c38c1bcf1fe9adf57eca6d4bb603428299b92f2f mkdir build cd build From 8b334f718f13fd1ab69be51c5dd331ca161c2553 Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Sat, 17 Dec 2022 03:19:20 +0200 Subject: [PATCH 35/36] wip --- scripts/pyperf_build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/pyperf_build.sh b/scripts/pyperf_build.sh index 46a0218b5..2f577a531 100755 --- a/scripts/pyperf_build.sh +++ b/scripts/pyperf_build.sh @@ -5,7 +5,7 @@ # set -euo pipefail -git clone --depth 1 -b aarch64 https://github.com/Granulate/bcc.git && cd bcc && git reset --hard c38c1bcf1fe9adf57eca6d4bb603428299b92f2f +git clone --depth 1 -b aarch64 https://github.com/Granulate/bcc.git && cd bcc && git reset --hard 846228b6e3f244732783540e3e0b868271902e2d mkdir build cd build From bbd17d1d40ca9e6c34f26d04ab4dcdc3374299d6 Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Sat, 17 Dec 2022 03:20:13 +0200 Subject: [PATCH 36/36] Update pyi.Dockerfile --- pyi.Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/pyi.Dockerfile b/pyi.Dockerfile index d399dbd8a..bba13a883 100644 --- a/pyi.Dockerfile +++ b/pyi.Dockerfile @@ -212,7 +212,6 @@ COPY ./scripts/pyperf_build.sh . RUN if [ "$(uname -m)" = "aarch64" ]; then v="7.0"; else v="7"; fi && \ source scl_source enable devtoolset-8 "llvm-toolset-$v" && \ source ./pyperf_build.sh exe - # gProfiler part WORKDIR /app