From 4bd758a203adb13c41f4c3c0aefae8ffcf5e281f Mon Sep 17 00:00:00 2001 From: Nicolas Lopez Date: Mon, 9 Dec 2024 12:45:15 -0500 Subject: [PATCH 01/14] add buck2 RE rust example --- buck2/rust/.buckconfig | 32 ++++++++++++ buck2/rust/.buckroot | 0 buck2/rust/.gitignore | 1 + buck2/rust/BUCK | 59 ++++++++++++++++++++++ buck2/rust/README.md | 58 +++++++++++++++++++++ buck2/rust/bin/main.rs | 3 ++ buck2/rust/platforms/BUCK | 32 ++++++++++++ buck2/rust/platforms/defs.bzl | 74 +++++++++++++++++++++++++++ buck2/rust/src/lib.rs | 3 ++ buck2/rust/test/test.rs | 4 ++ buck2/rust/toolchains/BUCK | 70 ++++++++++++++++++++++++++ buck2/rust/toolchains/defs.bzl | 92 ++++++++++++++++++++++++++++++++++ infra/test-buck2.sh | 6 +++ 13 files changed, 434 insertions(+) create mode 100644 buck2/rust/.buckconfig create mode 100644 buck2/rust/.buckroot create mode 100644 buck2/rust/.gitignore create mode 100644 buck2/rust/BUCK create mode 100644 buck2/rust/README.md create mode 100644 buck2/rust/bin/main.rs create mode 100644 buck2/rust/platforms/BUCK create mode 100644 buck2/rust/platforms/defs.bzl create mode 100644 buck2/rust/src/lib.rs create mode 100644 buck2/rust/test/test.rs create mode 100644 buck2/rust/toolchains/BUCK create mode 100644 buck2/rust/toolchains/defs.bzl diff --git a/buck2/rust/.buckconfig b/buck2/rust/.buckconfig new file mode 100644 index 00000000..8e773e99 --- /dev/null +++ b/buck2/rust/.buckconfig @@ -0,0 +1,32 @@ +[cells] + root = . + prelude = prelude + toolchains = toolchains + none = none + +[cell_aliases] + config = prelude + fbcode = none + fbsource = none + buck = none + +[external_cells] + prelude = bundled + +[parser] + target_platform_detector_spec = target:root//...->prelude//platforms:default + +[buck2] +digest_algorithms = SHA256 + +[buck2_re_client] +engine_address = .cluster.engflow.com +action_cache_address = .cluster.engflow.com +cas_address = .cluster.engflow.com +http_headers = + +[build] + execution_platforms = root//platforms:remote_platform + +[project] + ignore = .git diff --git a/buck2/rust/.buckroot b/buck2/rust/.buckroot new file mode 100644 index 00000000..e69de29b diff --git a/buck2/rust/.gitignore b/buck2/rust/.gitignore new file mode 100644 index 00000000..0a0ddb2e --- /dev/null +++ b/buck2/rust/.gitignore @@ -0,0 +1 @@ +/buck-out diff --git a/buck2/rust/BUCK b/buck2/rust/BUCK new file mode 100644 index 00000000..327c6671 --- /dev/null +++ b/buck2/rust/BUCK @@ -0,0 +1,59 @@ +# Copyright 2022 EngFlow Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +rust_library( + name = "library", + # We set these env vars to configure the RE Rust toolchain. Unfortunately the prelude does not provide a way to set these at the rust_toolchain level. We override the HOME location so rustp creates its `.rustp` directory in this location. + env = { + "RUSTUP_TOOLCHAIN":"/usr/local/rustup/toolchains/1.83.0-x86_64-unknown-linux-gnu", + "HOME": "buck-out/", + }, + srcs = glob( + ["src/**/*.rs"], + ), + exec_compatible_with = ["//platforms:remote_platform"], + default_target_platform = "//platforms:remote_platform", +) + +rust_binary( + name = "main", + # We set these env vars to configure the RE Rust toolchain. Unfortunately the prelude does not provide a way to set these at the rust_toolchain level. We override the HOME location so rustp creates its `.rustp` directory in this location. + env = { + "RUSTUP_TOOLCHAIN":"/usr/local/rustup/toolchains/1.83.0-x86_64-unknown-linux-gnu", + "HOME": "buck-out/", + }, + srcs = glob( + ["bin/**/*.rs"], + ), + crate_root = "bin/main.rs", + deps = [":library"], + exec_compatible_with = ["//platforms:remote_platform"], + default_target_platform = "//platforms:remote_platform", +) + +rust_test( + name = "test", + srcs = glob( + ["test/**/*.rs"], + ), + deps = [":library"], + # We set these env vars to configure the RE Rust toolchain. Unfortunately the prelude does not provide a way to set these at the rust_toolchain level. We override the HOME location so rustp creates its `.rustp` directory in this location. + env = { + "RUSTUP_TOOLCHAIN":"/usr/local/rustup/toolchains/1.83.0-x86_64-unknown-linux-gnu", + "HOME": "buck-out/", + }, + exec_compatible_with = ["//platforms:remote_platform"], + default_target_platform = "//platforms:remote_platform", + remote_execution_action_key_providers = "//platforms:remote_execution_action_keys", +) \ No newline at end of file diff --git a/buck2/rust/README.md b/buck2/rust/README.md new file mode 100644 index 00000000..8ed2249f --- /dev/null +++ b/buck2/rust/README.md @@ -0,0 +1,58 @@ +# EngFlow RE + Buck2 Rust example + +This example demonstrates use of EngFlow RE for a simple Rust project built with [Buck2](https://github.com/facebook/buck2) using the prelude. + +It is based on two existing samples in the Buck2 upstream repo: + +* Simple rust project with prelude - https://github.com/facebook/buck2/tree/main/examples/with_prelude/rust +* Buck2 remote execution integration with EngFlow - https://github.com/facebook/buck2/tree/main/examples/remote_execution/engflow + +### Example structure + +In the `platforms` cell we specify: +In the `platforms` cell we specify: +* The platform used for remote execution in this project `root//platforms:remote_platform`, which includes the definition of the Docker image used for remote execution, and that defines constraints for targets to run in the remote execution environment. This platform provides an `ExecutionPlatformRegistrationInfo` a `ConfigurationInfo` and a `PlatformInfo` to be able to be used in the `.buckconfig`, and in the `exec_compatible_with` and `default_target_platform` of `rust_*` rules. Note since rust rules depend on C++ linking this is needed similarly to the Buck2 cpp example in this repo. +* The action keys `root//platforms:remote_execution_action_keys`, which provides a default `BuildModeInfo` that is needed for RE of tests to function properly. +* The platform `image` configured in `platforms/defs.bzl`, notably, uses a different image than other Buck2 samples in this repo. Specifically, it uses a public AWS image that has `rust` preinstalled. This is because Buck2 rust rules do not include a hermetic rust toolchain. + +In the `toolchains` cell we specify: + +* The remote rust toolchain `root//toolchains:remote_rust_toolchain` which is compatible with the remote execution environment. This toolchain is configured with the `rustc_target_triple` to match the remote execution environment. +* The c++ toolchain `root//toolchains:cxx_tools_info_toolchain` that is compatible with the remote execution environment. +* The clang tools, `root//toolchains:path_clang_tools`, which is used by the c++ toolchain, and specifies the tools installed in the Docker image. +* The remote test execution toolchain, `root//toolchains:remote_test_execution_toolchain`. This toolchain defines platform options in the form of `capabilities`. Critically these include the `container-image`. This toolchain is identical to the one in the `buck2/cpp` sample in this repo. + +The `src`, `bin` and `test` cells: + +* Contain a copied version of https://github.com/facebook/buck2/tree/main/examples/with_prelude/rust that works with Buck2 and RE as configured in this sample project. + +* Key changes for remote execution in the top level `BUCK` file include setting environment variables to select the pre-installed rust toolchain in the container and set a custom value for `HOME`. This custom value is needed as `rustp` attempts to create a directory in the `HOME` location and we prefer this happen inside the `buck-out/` directory to guarantee actions do not modify any content outside of their scratch directory. + +To test the project with RE run (after setting up `.buckconfig` as indicated below): + +``` +buck2 test //:test +``` + +You can also build the `main` for this sample by running: + +``` +buck2 build //:main +``` + +### Relevant configs in `.buckconfig` + +The EngFlow endpoint and certificate should be configured as the +following: + +```ini +[buck2_re_client] +engine_address = .cluster.engflow.com +action_cache_address = .cluster.engflow.com +cas_address = .cluster.engflow.com +http_headers = + ``` + +To obtain the value of ``, log into https://.cluster.engflow.com/gettingstarted and obtain the value of `x-engflow-auth-token` in section `Method 2: JWT`, take note of this value. Then set `AUTH_HTTP_HEADERS` with the value `x-engflow-auth-method:jwt-v0,x-engflow-auth-token:. + +Note for CI runs, the auth method used is [Github Tokens](https://docs.engflow.com/re/config/authentication.html#github-tokens). diff --git a/buck2/rust/bin/main.rs b/buck2/rust/bin/main.rs new file mode 100644 index 00000000..c0e2f814 --- /dev/null +++ b/buck2/rust/bin/main.rs @@ -0,0 +1,3 @@ +fn main() { + library::print_hello(); +} \ No newline at end of file diff --git a/buck2/rust/platforms/BUCK b/buck2/rust/platforms/BUCK new file mode 100644 index 00000000..2fc7a736 --- /dev/null +++ b/buck2/rust/platforms/BUCK @@ -0,0 +1,32 @@ +# Copyright 2022 EngFlow Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load(":defs.bzl", "platforms") +load(":defs.bzl", "action_keys") + +# This platform configures details of remote execution. +platforms( + name = "remote_platform", + cpu_configuration = "config//cpu:x86_64", + os_configuration = "config//os:linux", +) + +# This action_key provides a default BuildModeInfo that is needed for RE of tests to function properly. +# The values in `cell` and `mode` can be used, in practice, to create cache silos. Any values can be given to these attributes. +action_keys( + name = "remote_execution_action_keys", + cell = "standard", + mode = "standard", + visibility = ["PUBLIC"], +) \ No newline at end of file diff --git a/buck2/rust/platforms/defs.bzl b/buck2/rust/platforms/defs.bzl new file mode 100644 index 00000000..ac30d696 --- /dev/null +++ b/buck2/rust/platforms/defs.bzl @@ -0,0 +1,74 @@ +# Copyright 2022 EngFlow Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This platform is essentially the same as the one provided in https://github.com/facebook/buck2/blob/804d62242214455d51787f7c8c96a1e12c75ec32/examples/remote_execution/engflow/platforms/defs.bzl +# The main difference is we enable passing CPU and OS constraints and we use the sample EngFlow RE image. +load("@prelude//:build_mode.bzl", "BuildModeInfo") + +def _platforms(ctx): + constraints = dict() + constraints.update(ctx.attrs.cpu_configuration[ConfigurationInfo].constraints) + constraints.update(ctx.attrs.os_configuration[ConfigurationInfo].constraints) + configuration = ConfigurationInfo( + constraints = constraints, + values = {}, + ) + + # A bookworm image with rust pre-installed. + image = "docker://public.ecr.aws/docker/library/rust:1.83.0-bullseye@sha256:24118f76a7da011b22a25b8e9dbdbb549ed29c1eba635d6aa4a9c9f5ed545066" + name = ctx.label.raw_target() + platform = ExecutionPlatformInfo( + label = ctx.label.raw_target(), + configuration = configuration, + executor_config = CommandExecutorConfig( + local_enabled = False, + remote_enabled = True, + use_limited_hybrid = False, + remote_execution_properties = { + "container-image": image, + }, + remote_execution_use_case = "buck2-default", + # TODO: Use output_paths + remote_output_paths = "strict", + ), + ) + + return [ + DefaultInfo(), + ExecutionPlatformRegistrationInfo(platforms = [platform]), + configuration, + PlatformInfo(label = str(name), configuration = configuration), + ] + +def _action_keys(ctx): + return [ + DefaultInfo(), + BuildModeInfo(cell = ctx.attrs.cell, mode = ctx.attrs.mode), + ] + +platforms = rule( + attrs = { + "cpu_configuration": attrs.dep(providers = [ConfigurationInfo]), + "os_configuration": attrs.dep(providers = [ConfigurationInfo]), + }, + impl = _platforms +) + +action_keys = rule( + attrs = { + "cell": attrs.string(), + "mode": attrs.string(), + }, + impl = _action_keys +) \ No newline at end of file diff --git a/buck2/rust/src/lib.rs b/buck2/rust/src/lib.rs new file mode 100644 index 00000000..3c124177 --- /dev/null +++ b/buck2/rust/src/lib.rs @@ -0,0 +1,3 @@ +pub fn print_hello() { + println!("hello world from rust toolchain"); +} \ No newline at end of file diff --git a/buck2/rust/test/test.rs b/buck2/rust/test/test.rs new file mode 100644 index 00000000..cf97a8b0 --- /dev/null +++ b/buck2/rust/test/test.rs @@ -0,0 +1,4 @@ +#[test] +fn test_it_doesnt_crash() { + library::print_hello(); +} \ No newline at end of file diff --git a/buck2/rust/toolchains/BUCK b/buck2/rust/toolchains/BUCK new file mode 100644 index 00000000..2a558930 --- /dev/null +++ b/buck2/rust/toolchains/BUCK @@ -0,0 +1,70 @@ + +# Copyright 2022 EngFlow Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("defs.bzl", "remote_rust_toolchain", "path_clang_tools") +load("@prelude//toolchains:cxx.bzl", "cxx_tools_info_toolchain") +load("@prelude//toolchains:python.bzl", "system_python_bootstrap_toolchain") +load("@prelude//toolchains:remote_test_execution.bzl", "remote_test_execution_toolchain") +load("@prelude//tests:test_toolchain.bzl", "noop_test_toolchain") + +remote_rust_toolchain( + name = "rust", + default_edition = "2021", + visibility = ["PUBLIC"], +) + +# Python toolchain used in scripts that bootstrap other aspects of the Buck2 prelude. +system_python_bootstrap_toolchain( + name = "python_bootstrap", + visibility = ["PUBLIC"], +) + +# Custom clang tools that use g++ for linking. +path_clang_tools( + name = "clang_tools", + #target_compatible_with = ["config//os:linux"], + visibility = ["PUBLIC"], +) + +# Custom cpp toolchain that is compatible with the remote worker environment. +cxx_tools_info_toolchain( + name = "cxx", + #target_compatible_with = ["config//os:linux"], + cxx_tools_info = ":clang_tools", + visibility = ["PUBLIC"], +) + +# Default toolchain for remote execution of tests. +# Note it defines a profile with a capability that defines the `container-image` that matches the one defined in //platforms:remote_platform. +# Capabilities are passed to the RE service to find workers that match them as Platform options. +remote_test_execution_toolchain( + name = "remote_test_execution", + visibility = ["PUBLIC"], + default_profile = "cxx_re_toolchain", + profiles = { + "cxx_re_toolchain": { + "use_case": "cxx-testing", + "capabilities": { + "container-image" : "docker://gcr.io/bazel-public/ubuntu2004-java11@sha256:69a78f121230c6d5cbfe2f4af8ce65481aa3f2acaaaf8e899df335f1ac1b35b5", + }, + } + }, +) + +# In some cases the execution of test can fail looking for this `noop_test_toolchain`. +noop_test_toolchain( + name = "test", + visibility = ["PUBLIC"], +) diff --git a/buck2/rust/toolchains/defs.bzl b/buck2/rust/toolchains/defs.bzl new file mode 100644 index 00000000..5ecdd0e5 --- /dev/null +++ b/buck2/rust/toolchains/defs.bzl @@ -0,0 +1,92 @@ +# Copyright 2022 EngFlow Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@prelude//rust:rust_toolchain.bzl", "PanicRuntime", "RustToolchainInfo") +load("@prelude//rust/tools:attrs.bzl", "internal_tool_attrs") +load("@prelude//cxx:cxx_toolchain_types.bzl", "LinkerType") +load("@prelude//toolchains:cxx.bzl", "CxxToolsInfo") + +# This def is similar to the one in https://github.com/facebook/buck2/blob/804d62242214455d51787f7c8c96a1e12c75ec32/prelude/toolchains/cxx/clang/tools.bzl +# The main difference is we set linker to "g++" to match the cpp tools installed in the sample docker container. +def _path_clang_tools_impl(_ctx) -> list[Provider]: + return [ + DefaultInfo(), + CxxToolsInfo( + compiler = "clang", + compiler_type = "clang", + cxx_compiler = "clang++", + asm_compiler = "clang", + asm_compiler_type = "clang", + rc_compiler = None, + cvtres_compiler = None, + archiver = "ar", + archiver_type = "gnu", + linker = "g++", + linker_type = LinkerType("gnu"), + ), + ] + +path_clang_tools = rule( + impl = _path_clang_tools_impl, + attrs = {}, +) + +def _remote_rust_toolchain_impl(ctx): + return [ + DefaultInfo(), + RustToolchainInfo( + allow_lints = ctx.attrs.allow_lints, + clippy_driver = RunInfo(args = ["clippy-driver"]), + clippy_toml = ctx.attrs.clippy_toml[DefaultInfo].default_outputs[0] if ctx.attrs.clippy_toml else None, + compiler = RunInfo(args = ["rustc"]), + default_edition = ctx.attrs.default_edition, + panic_runtime = PanicRuntime("unwind"), + deny_lints = ctx.attrs.deny_lints, + doctests = ctx.attrs.doctests, + failure_filter_action = ctx.attrs.failure_filter_action[RunInfo], + nightly_features = ctx.attrs.nightly_features, + report_unused_deps = ctx.attrs.report_unused_deps, + rustc_action = ctx.attrs.rustc_action[RunInfo], + rustc_binary_flags = ctx.attrs.rustc_binary_flags, + rustc_flags = ctx.attrs.rustc_flags, + rustc_target_triple = ctx.attrs.rustc_target_triple, + rustc_test_flags = ctx.attrs.rustc_test_flags, + rustdoc = RunInfo(args = ["rustdoc"]), + rustdoc_flags = ctx.attrs.rustdoc_flags, + rustdoc_test_with_resources = ctx.attrs.rustdoc_test_with_resources[RunInfo], + rustdoc_coverage = ctx.attrs.rustdoc_coverage[RunInfo], + transitive_dependency_symlinks_tool = ctx.attrs.transitive_dependency_symlinks_tool[RunInfo], + warn_lints = ctx.attrs.warn_lints, + ), + ] + +remote_rust_toolchain = rule( + impl = _remote_rust_toolchain_impl, + attrs = internal_tool_attrs | { + "allow_lints": attrs.list(attrs.string(), default = []), + "clippy_toml": attrs.option(attrs.dep(providers = [DefaultInfo]), default = None), + "default_edition": attrs.option(attrs.string(), default = None), + "deny_lints": attrs.list(attrs.string(), default = []), + "doctests": attrs.bool(default = False), + "nightly_features": attrs.bool(default = False), + "report_unused_deps": attrs.bool(default = False), + "rustc_binary_flags": attrs.list(attrs.string(), default = []), + "rustc_flags": attrs.list(attrs.string(), default = []), + "rustc_target_triple": attrs.string(default = "x86_64-unknown-linux-gnu"), + "rustc_test_flags": attrs.list(attrs.string(), default = []), + "rustdoc_flags": attrs.list(attrs.string(), default = []), + "warn_lints": attrs.list(attrs.string(), default = []), + }, + is_toolchain_rule = True, +) diff --git a/infra/test-buck2.sh b/infra/test-buck2.sh index 710c9bba..6c06df02 100755 --- a/infra/test-buck2.sh +++ b/infra/test-buck2.sh @@ -21,3 +21,9 @@ cd golang buck2 build //go:hello buck2 test //go/greeting:greeting_test cd .. + +# Run rust example +cd rust +buck2 build //:main +buck2 test //:test +cd .. From b72719b5fe8914bf8ee715a8aaba15258a186c42 Mon Sep 17 00:00:00 2001 From: Nicolas Lopez Date: Mon, 9 Dec 2024 12:47:38 -0500 Subject: [PATCH 02/14] nits --- buck2/rust/BUCK | 2 +- buck2/rust/bin/main.rs | 2 +- buck2/rust/platforms/BUCK | 2 +- buck2/rust/platforms/defs.bzl | 2 +- buck2/rust/src/lib.rs | 2 +- buck2/rust/test/test.rs | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/buck2/rust/BUCK b/buck2/rust/BUCK index 327c6671..672fa7dc 100644 --- a/buck2/rust/BUCK +++ b/buck2/rust/BUCK @@ -56,4 +56,4 @@ rust_test( exec_compatible_with = ["//platforms:remote_platform"], default_target_platform = "//platforms:remote_platform", remote_execution_action_key_providers = "//platforms:remote_execution_action_keys", -) \ No newline at end of file +) diff --git a/buck2/rust/bin/main.rs b/buck2/rust/bin/main.rs index c0e2f814..54a4b8bf 100644 --- a/buck2/rust/bin/main.rs +++ b/buck2/rust/bin/main.rs @@ -1,3 +1,3 @@ fn main() { library::print_hello(); -} \ No newline at end of file +} diff --git a/buck2/rust/platforms/BUCK b/buck2/rust/platforms/BUCK index 2fc7a736..8e4f659a 100644 --- a/buck2/rust/platforms/BUCK +++ b/buck2/rust/platforms/BUCK @@ -29,4 +29,4 @@ action_keys( cell = "standard", mode = "standard", visibility = ["PUBLIC"], -) \ No newline at end of file +) diff --git a/buck2/rust/platforms/defs.bzl b/buck2/rust/platforms/defs.bzl index ac30d696..29deb6e6 100644 --- a/buck2/rust/platforms/defs.bzl +++ b/buck2/rust/platforms/defs.bzl @@ -71,4 +71,4 @@ action_keys = rule( "mode": attrs.string(), }, impl = _action_keys -) \ No newline at end of file +) diff --git a/buck2/rust/src/lib.rs b/buck2/rust/src/lib.rs index 3c124177..84c45251 100644 --- a/buck2/rust/src/lib.rs +++ b/buck2/rust/src/lib.rs @@ -1,3 +1,3 @@ pub fn print_hello() { println!("hello world from rust toolchain"); -} \ No newline at end of file +} diff --git a/buck2/rust/test/test.rs b/buck2/rust/test/test.rs index cf97a8b0..c5cdb345 100644 --- a/buck2/rust/test/test.rs +++ b/buck2/rust/test/test.rs @@ -1,4 +1,4 @@ #[test] fn test_it_doesnt_crash() { library::print_hello(); -} \ No newline at end of file +} From a57d110118e60611dafd92e9f0b2b5ac6e063e7c Mon Sep 17 00:00:00 2001 From: Nicolas Lopez Date: Mon, 9 Dec 2024 15:12:34 -0500 Subject: [PATCH 03/14] fix detail in toolchain profile --- buck2/rust/toolchains/BUCK | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buck2/rust/toolchains/BUCK b/buck2/rust/toolchains/BUCK index 2a558930..f0010a95 100644 --- a/buck2/rust/toolchains/BUCK +++ b/buck2/rust/toolchains/BUCK @@ -57,7 +57,7 @@ remote_test_execution_toolchain( "cxx_re_toolchain": { "use_case": "cxx-testing", "capabilities": { - "container-image" : "docker://gcr.io/bazel-public/ubuntu2004-java11@sha256:69a78f121230c6d5cbfe2f4af8ce65481aa3f2acaaaf8e899df335f1ac1b35b5", + "container-image" : "docker://public.ecr.aws/docker/library/rust:1.83.0-bullseye@sha256:24118f76a7da011b22a25b8e9dbdbb549ed29c1eba635d6aa4a9c9f5ed545066", }, } }, From ab9990b387a69a353fc7c1205f4bc0dc92787555 Mon Sep 17 00:00:00 2001 From: Nicolas Lopez Date: Tue, 10 Dec 2024 15:22:27 -0500 Subject: [PATCH 04/14] use select statements for RE attrs --- buck2/rust/BUCK | 58 +++++++++++++++++++++++------------ buck2/rust/README.md | 4 +-- buck2/rust/platforms/BUCK | 12 +++++++- buck2/rust/platforms/defs.bzl | 4 ++- infra/test-buck2.sh | 4 +-- 5 files changed, 56 insertions(+), 26 deletions(-) diff --git a/buck2/rust/BUCK b/buck2/rust/BUCK index 672fa7dc..7cf8e35b 100644 --- a/buck2/rust/BUCK +++ b/buck2/rust/BUCK @@ -15,31 +15,41 @@ rust_library( name = "library", # We set these env vars to configure the RE Rust toolchain. Unfortunately the prelude does not provide a way to set these at the rust_toolchain level. We override the HOME location so rustp creates its `.rustp` directory in this location. - env = { - "RUSTUP_TOOLCHAIN":"/usr/local/rustup/toolchains/1.83.0-x86_64-unknown-linux-gnu", - "HOME": "buck-out/", - }, + env = select({ + "//platforms:engflow": { + "RUSTUP_TOOLCHAIN":"/usr/local/rustup/toolchains/1.83.0-x86_64-unknown-linux-gnu", + "HOME": "buck-out/", + }, + "DEFAULT": {}, + }), srcs = glob( ["src/**/*.rs"], ), - exec_compatible_with = ["//platforms:remote_platform"], - default_target_platform = "//platforms:remote_platform", + exec_compatible_with = select({ + "//platforms:engflow": ["//platforms:remote_platform"], + "DEFAULT": [], + }), ) rust_binary( name = "main", # We set these env vars to configure the RE Rust toolchain. Unfortunately the prelude does not provide a way to set these at the rust_toolchain level. We override the HOME location so rustp creates its `.rustp` directory in this location. - env = { - "RUSTUP_TOOLCHAIN":"/usr/local/rustup/toolchains/1.83.0-x86_64-unknown-linux-gnu", - "HOME": "buck-out/", - }, + env = select({ + "//platforms:engflow": { + "RUSTUP_TOOLCHAIN":"/usr/local/rustup/toolchains/1.83.0-x86_64-unknown-linux-gnu", + "HOME": "buck-out/", + }, + "DEFAULT": {}, + }), srcs = glob( ["bin/**/*.rs"], ), crate_root = "bin/main.rs", deps = [":library"], - exec_compatible_with = ["//platforms:remote_platform"], - default_target_platform = "//platforms:remote_platform", + exec_compatible_with = select({ + "//platforms:engflow": ["//platforms:remote_platform"], + "DEFAULT": [], + }), ) rust_test( @@ -49,11 +59,19 @@ rust_test( ), deps = [":library"], # We set these env vars to configure the RE Rust toolchain. Unfortunately the prelude does not provide a way to set these at the rust_toolchain level. We override the HOME location so rustp creates its `.rustp` directory in this location. - env = { - "RUSTUP_TOOLCHAIN":"/usr/local/rustup/toolchains/1.83.0-x86_64-unknown-linux-gnu", - "HOME": "buck-out/", - }, - exec_compatible_with = ["//platforms:remote_platform"], - default_target_platform = "//platforms:remote_platform", - remote_execution_action_key_providers = "//platforms:remote_execution_action_keys", -) + env = select({ + "//platforms:engflow": { + "RUSTUP_TOOLCHAIN":"/usr/local/rustup/toolchains/1.83.0-x86_64-unknown-linux-gnu", + "HOME": "buck-out/", + }, + "DEFAULT": {}, + }), + exec_compatible_with = select({ + "//platforms:engflow": ["//platforms:remote_platform"], + "DEFAULT": [], + }), + remote_execution_action_key_providers = select({ + "//platforms:engflow": "//platforms:remote_execution_action_keys", + "DEFAULT": None, + }), +) \ No newline at end of file diff --git a/buck2/rust/README.md b/buck2/rust/README.md index 8ed2249f..4c44c109 100644 --- a/buck2/rust/README.md +++ b/buck2/rust/README.md @@ -31,13 +31,13 @@ The `src`, `bin` and `test` cells: To test the project with RE run (after setting up `.buckconfig` as indicated below): ``` -buck2 test //:test +buck2 test --target-platforms //platforms:remote_platform //:test ``` You can also build the `main` for this sample by running: ``` -buck2 build //:main +buck2 build --target-platforms //platforms:remote_platform //:main ``` ### Relevant configs in `.buckconfig` diff --git a/buck2/rust/platforms/BUCK b/buck2/rust/platforms/BUCK index 8e4f659a..2c280830 100644 --- a/buck2/rust/platforms/BUCK +++ b/buck2/rust/platforms/BUCK @@ -15,11 +15,21 @@ load(":defs.bzl", "platforms") load(":defs.bzl", "action_keys") +constraint_setting( + name = "re_provider" +) + +constraint_value( + name = "engflow", + constraint_setting = ":re_provider", +) + # This platform configures details of remote execution. platforms( name = "remote_platform", cpu_configuration = "config//cpu:x86_64", os_configuration = "config//os:linux", + re_provider = ":engflow", ) # This action_key provides a default BuildModeInfo that is needed for RE of tests to function properly. @@ -29,4 +39,4 @@ action_keys( cell = "standard", mode = "standard", visibility = ["PUBLIC"], -) +) \ No newline at end of file diff --git a/buck2/rust/platforms/defs.bzl b/buck2/rust/platforms/defs.bzl index 29deb6e6..ed118466 100644 --- a/buck2/rust/platforms/defs.bzl +++ b/buck2/rust/platforms/defs.bzl @@ -20,6 +20,7 @@ def _platforms(ctx): constraints = dict() constraints.update(ctx.attrs.cpu_configuration[ConfigurationInfo].constraints) constraints.update(ctx.attrs.os_configuration[ConfigurationInfo].constraints) + constraints.update(ctx.attrs.re_provider[ConfigurationInfo].constraints) configuration = ConfigurationInfo( constraints = constraints, values = {}, @@ -61,6 +62,7 @@ platforms = rule( attrs = { "cpu_configuration": attrs.dep(providers = [ConfigurationInfo]), "os_configuration": attrs.dep(providers = [ConfigurationInfo]), + "re_provider": attrs.dep(providers = [ConfigurationInfo]), }, impl = _platforms ) @@ -71,4 +73,4 @@ action_keys = rule( "mode": attrs.string(), }, impl = _action_keys -) +) \ No newline at end of file diff --git a/infra/test-buck2.sh b/infra/test-buck2.sh index 6c06df02..e5ee92b2 100755 --- a/infra/test-buck2.sh +++ b/infra/test-buck2.sh @@ -24,6 +24,6 @@ cd .. # Run rust example cd rust -buck2 build //:main -buck2 test //:test +buck2 build --target-platforms //platforms:remote_platform //:main +buck2 test --target-platforms //platforms:remote_platform //:test cd .. From 986e0ab0bc71fb6f64cfd42162f2ed8d1f2da6c2 Mon Sep 17 00:00:00 2001 From: Nicolas Lopez Date: Tue, 10 Dec 2024 15:47:48 -0500 Subject: [PATCH 05/14] nits --- buck2/rust/BUCK | 2 +- buck2/rust/README.md | 4 +++- buck2/rust/platforms/BUCK | 2 +- buck2/rust/platforms/defs.bzl | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/buck2/rust/BUCK b/buck2/rust/BUCK index 7cf8e35b..dd3a91a2 100644 --- a/buck2/rust/BUCK +++ b/buck2/rust/BUCK @@ -74,4 +74,4 @@ rust_test( "//platforms:engflow": "//platforms:remote_execution_action_keys", "DEFAULT": None, }), -) \ No newline at end of file +) diff --git a/buck2/rust/README.md b/buck2/rust/README.md index 4c44c109..e03ea0c2 100644 --- a/buck2/rust/README.md +++ b/buck2/rust/README.md @@ -11,7 +11,7 @@ It is based on two existing samples in the Buck2 upstream repo: In the `platforms` cell we specify: In the `platforms` cell we specify: -* The platform used for remote execution in this project `root//platforms:remote_platform`, which includes the definition of the Docker image used for remote execution, and that defines constraints for targets to run in the remote execution environment. This platform provides an `ExecutionPlatformRegistrationInfo` a `ConfigurationInfo` and a `PlatformInfo` to be able to be used in the `.buckconfig`, and in the `exec_compatible_with` and `default_target_platform` of `rust_*` rules. Note since rust rules depend on C++ linking this is needed similarly to the Buck2 cpp example in this repo. +* The platform used for remote execution in this project `root//platforms:remote_platform`, which includes the definition of the Docker image used for remote execution, and that defines constraints for targets to run in the remote execution environment. This platform provides an `ExecutionPlatformRegistrationInfo` a `ConfigurationInfo` and a `PlatformInfo` to be able to be used in the `.buckconfig`, in the `--target-platforms` flag, and in the `exec_compatible_with` of `rust_*` rules. Note since rust rules depend on C++ linking this is needed similarly to the Buck2 cpp example in this repo. * The action keys `root//platforms:remote_execution_action_keys`, which provides a default `BuildModeInfo` that is needed for RE of tests to function properly. * The platform `image` configured in `platforms/defs.bzl`, notably, uses a different image than other Buck2 samples in this repo. Specifically, it uses a public AWS image that has `rust` preinstalled. This is because Buck2 rust rules do not include a hermetic rust toolchain. @@ -40,6 +40,8 @@ You can also build the `main` for this sample by running: buck2 build --target-platforms //platforms:remote_platform //:main ``` +Note the use of `--target-platforms` to select the remote platform as part of the build / test command. + ### Relevant configs in `.buckconfig` The EngFlow endpoint and certificate should be configured as the diff --git a/buck2/rust/platforms/BUCK b/buck2/rust/platforms/BUCK index 2c280830..67a4d912 100644 --- a/buck2/rust/platforms/BUCK +++ b/buck2/rust/platforms/BUCK @@ -39,4 +39,4 @@ action_keys( cell = "standard", mode = "standard", visibility = ["PUBLIC"], -) \ No newline at end of file +) diff --git a/buck2/rust/platforms/defs.bzl b/buck2/rust/platforms/defs.bzl index ed118466..7b4b352d 100644 --- a/buck2/rust/platforms/defs.bzl +++ b/buck2/rust/platforms/defs.bzl @@ -73,4 +73,4 @@ action_keys = rule( "mode": attrs.string(), }, impl = _action_keys -) \ No newline at end of file +) From 1e52fce9fef4fe9b16a5340a47cad74fdb456fe6 Mon Sep 17 00:00:00 2001 From: Nicolas Lopez Date: Thu, 12 Dec 2024 13:45:54 -0500 Subject: [PATCH 06/14] override RUSTUP_HOME not HOME and point to image details --- buck2/rust/BUCK | 6 +++--- buck2/rust/platforms/defs.bzl | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/buck2/rust/BUCK b/buck2/rust/BUCK index dd3a91a2..9190f234 100644 --- a/buck2/rust/BUCK +++ b/buck2/rust/BUCK @@ -18,7 +18,7 @@ rust_library( env = select({ "//platforms:engflow": { "RUSTUP_TOOLCHAIN":"/usr/local/rustup/toolchains/1.83.0-x86_64-unknown-linux-gnu", - "HOME": "buck-out/", + "RUSTUP_HOME": "buck-out/.rustup", }, "DEFAULT": {}, }), @@ -37,7 +37,7 @@ rust_binary( env = select({ "//platforms:engflow": { "RUSTUP_TOOLCHAIN":"/usr/local/rustup/toolchains/1.83.0-x86_64-unknown-linux-gnu", - "HOME": "buck-out/", + "RUSTUP_HOME": "buck-out/.rustup", }, "DEFAULT": {}, }), @@ -62,7 +62,7 @@ rust_test( env = select({ "//platforms:engflow": { "RUSTUP_TOOLCHAIN":"/usr/local/rustup/toolchains/1.83.0-x86_64-unknown-linux-gnu", - "HOME": "buck-out/", + "RUSTUP_HOME": "buck-out/.rustup", }, "DEFAULT": {}, }), diff --git a/buck2/rust/platforms/defs.bzl b/buck2/rust/platforms/defs.bzl index 7b4b352d..0b38e6c0 100644 --- a/buck2/rust/platforms/defs.bzl +++ b/buck2/rust/platforms/defs.bzl @@ -26,7 +26,7 @@ def _platforms(ctx): values = {}, ) - # A bookworm image with rust pre-installed. + # A bookworm image with rust pre-installed. Image details can be found in https://gallery.ecr.aws/docker/library/rust. image = "docker://public.ecr.aws/docker/library/rust:1.83.0-bullseye@sha256:24118f76a7da011b22a25b8e9dbdbb549ed29c1eba635d6aa4a9c9f5ed545066" name = ctx.label.raw_target() platform = ExecutionPlatformInfo( From 03772a81cb2af93b6111971d909b35bde6858108 Mon Sep 17 00:00:00 2001 From: Nicolas Lopez Date: Tue, 17 Dec 2024 11:45:47 -0500 Subject: [PATCH 07/14] get rustup home out of buck2 dirs --- buck2/rust/BUCK | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/buck2/rust/BUCK b/buck2/rust/BUCK index 9190f234..66840999 100644 --- a/buck2/rust/BUCK +++ b/buck2/rust/BUCK @@ -18,7 +18,7 @@ rust_library( env = select({ "//platforms:engflow": { "RUSTUP_TOOLCHAIN":"/usr/local/rustup/toolchains/1.83.0-x86_64-unknown-linux-gnu", - "RUSTUP_HOME": "buck-out/.rustup", + "RUSTUP_HOME": ".rustup", }, "DEFAULT": {}, }), @@ -37,7 +37,7 @@ rust_binary( env = select({ "//platforms:engflow": { "RUSTUP_TOOLCHAIN":"/usr/local/rustup/toolchains/1.83.0-x86_64-unknown-linux-gnu", - "RUSTUP_HOME": "buck-out/.rustup", + "RUSTUP_HOME": ".rustup", }, "DEFAULT": {}, }), @@ -62,7 +62,7 @@ rust_test( env = select({ "//platforms:engflow": { "RUSTUP_TOOLCHAIN":"/usr/local/rustup/toolchains/1.83.0-x86_64-unknown-linux-gnu", - "RUSTUP_HOME": "buck-out/.rustup", + "RUSTUP_HOME": ".rustup", }, "DEFAULT": {}, }), From b14757d553d9bb2c6d1a009a6dc32b7d09ca578f Mon Sep 17 00:00:00 2001 From: Nicolas Lopez Date: Thu, 19 Dec 2024 11:15:22 -0500 Subject: [PATCH 08/14] fix toolchain following Antonio's advice --- buck2/rust/BUCK | 36 ---------------------------------- buck2/rust/platforms/defs.bzl | 1 + buck2/rust/toolchains/BUCK | 6 ++++-- buck2/rust/toolchains/defs.bzl | 9 ++++++--- 4 files changed, 11 insertions(+), 41 deletions(-) diff --git a/buck2/rust/BUCK b/buck2/rust/BUCK index 66840999..3cc3dafa 100644 --- a/buck2/rust/BUCK +++ b/buck2/rust/BUCK @@ -14,42 +14,18 @@ rust_library( name = "library", - # We set these env vars to configure the RE Rust toolchain. Unfortunately the prelude does not provide a way to set these at the rust_toolchain level. We override the HOME location so rustp creates its `.rustp` directory in this location. - env = select({ - "//platforms:engflow": { - "RUSTUP_TOOLCHAIN":"/usr/local/rustup/toolchains/1.83.0-x86_64-unknown-linux-gnu", - "RUSTUP_HOME": ".rustup", - }, - "DEFAULT": {}, - }), srcs = glob( ["src/**/*.rs"], ), - exec_compatible_with = select({ - "//platforms:engflow": ["//platforms:remote_platform"], - "DEFAULT": [], - }), ) rust_binary( name = "main", - # We set these env vars to configure the RE Rust toolchain. Unfortunately the prelude does not provide a way to set these at the rust_toolchain level. We override the HOME location so rustp creates its `.rustp` directory in this location. - env = select({ - "//platforms:engflow": { - "RUSTUP_TOOLCHAIN":"/usr/local/rustup/toolchains/1.83.0-x86_64-unknown-linux-gnu", - "RUSTUP_HOME": ".rustup", - }, - "DEFAULT": {}, - }), srcs = glob( ["bin/**/*.rs"], ), crate_root = "bin/main.rs", deps = [":library"], - exec_compatible_with = select({ - "//platforms:engflow": ["//platforms:remote_platform"], - "DEFAULT": [], - }), ) rust_test( @@ -58,18 +34,6 @@ rust_test( ["test/**/*.rs"], ), deps = [":library"], - # We set these env vars to configure the RE Rust toolchain. Unfortunately the prelude does not provide a way to set these at the rust_toolchain level. We override the HOME location so rustp creates its `.rustp` directory in this location. - env = select({ - "//platforms:engflow": { - "RUSTUP_TOOLCHAIN":"/usr/local/rustup/toolchains/1.83.0-x86_64-unknown-linux-gnu", - "RUSTUP_HOME": ".rustup", - }, - "DEFAULT": {}, - }), - exec_compatible_with = select({ - "//platforms:engflow": ["//platforms:remote_platform"], - "DEFAULT": [], - }), remote_execution_action_key_providers = select({ "//platforms:engflow": "//platforms:remote_execution_action_keys", "DEFAULT": None, diff --git a/buck2/rust/platforms/defs.bzl b/buck2/rust/platforms/defs.bzl index 0b38e6c0..386a109e 100644 --- a/buck2/rust/platforms/defs.bzl +++ b/buck2/rust/platforms/defs.bzl @@ -27,6 +27,7 @@ def _platforms(ctx): ) # A bookworm image with rust pre-installed. Image details can be found in https://gallery.ecr.aws/docker/library/rust. + # Dockerfile can be found in https://github.com/rust-lang/docker-rust/blob/700c4f146427808cfb1e07a646e4afabbe99da4f/stable/bullseye/Dockerfile image = "docker://public.ecr.aws/docker/library/rust:1.83.0-bullseye@sha256:24118f76a7da011b22a25b8e9dbdbb549ed29c1eba635d6aa4a9c9f5ed545066" name = ctx.label.raw_target() platform = ExecutionPlatformInfo( diff --git a/buck2/rust/toolchains/BUCK b/buck2/rust/toolchains/BUCK index f0010a95..9fc11b68 100644 --- a/buck2/rust/toolchains/BUCK +++ b/buck2/rust/toolchains/BUCK @@ -23,6 +23,10 @@ remote_rust_toolchain( name = "rust", default_edition = "2021", visibility = ["PUBLIC"], + rustdoc = "/usr/local/rustup/toolchains/1.83.0-x86_64-unknown-linux-gnu/bin/rustc", + compiler = "/usr/local/rustup/toolchains/1.83.0-x86_64-unknown-linux-gnu/bin/rustc", + clippy_driver = "/usr/local/rustup/toolchains/1.83.0-x86_64-unknown-linux-gnu/bin/rustc", + rustc_target_triple = "x86_64-unknown-linux-gnu", ) # Python toolchain used in scripts that bootstrap other aspects of the Buck2 prelude. @@ -34,14 +38,12 @@ system_python_bootstrap_toolchain( # Custom clang tools that use g++ for linking. path_clang_tools( name = "clang_tools", - #target_compatible_with = ["config//os:linux"], visibility = ["PUBLIC"], ) # Custom cpp toolchain that is compatible with the remote worker environment. cxx_tools_info_toolchain( name = "cxx", - #target_compatible_with = ["config//os:linux"], cxx_tools_info = ":clang_tools", visibility = ["PUBLIC"], ) diff --git a/buck2/rust/toolchains/defs.bzl b/buck2/rust/toolchains/defs.bzl index 5ecdd0e5..e6b2a8f3 100644 --- a/buck2/rust/toolchains/defs.bzl +++ b/buck2/rust/toolchains/defs.bzl @@ -47,9 +47,9 @@ def _remote_rust_toolchain_impl(ctx): DefaultInfo(), RustToolchainInfo( allow_lints = ctx.attrs.allow_lints, - clippy_driver = RunInfo(args = ["clippy-driver"]), + clippy_driver = RunInfo(args = [ctx.attrs.clippy_driver]), clippy_toml = ctx.attrs.clippy_toml[DefaultInfo].default_outputs[0] if ctx.attrs.clippy_toml else None, - compiler = RunInfo(args = ["rustc"]), + compiler = RunInfo(args = [ctx.attrs.compiler]), default_edition = ctx.attrs.default_edition, panic_runtime = PanicRuntime("unwind"), deny_lints = ctx.attrs.deny_lints, @@ -62,7 +62,7 @@ def _remote_rust_toolchain_impl(ctx): rustc_flags = ctx.attrs.rustc_flags, rustc_target_triple = ctx.attrs.rustc_target_triple, rustc_test_flags = ctx.attrs.rustc_test_flags, - rustdoc = RunInfo(args = ["rustdoc"]), + rustdoc = RunInfo(args = [ctx.attrs.rustdoc]), rustdoc_flags = ctx.attrs.rustdoc_flags, rustdoc_test_with_resources = ctx.attrs.rustdoc_test_with_resources[RunInfo], rustdoc_coverage = ctx.attrs.rustdoc_coverage[RunInfo], @@ -75,7 +75,9 @@ remote_rust_toolchain = rule( impl = _remote_rust_toolchain_impl, attrs = internal_tool_attrs | { "allow_lints": attrs.list(attrs.string(), default = []), + "clippy_driver": attrs.string(default = "clippy-driver"), "clippy_toml": attrs.option(attrs.dep(providers = [DefaultInfo]), default = None), + "compiler": attrs.string(default = "rustc"), "default_edition": attrs.option(attrs.string(), default = None), "deny_lints": attrs.list(attrs.string(), default = []), "doctests": attrs.bool(default = False), @@ -85,6 +87,7 @@ remote_rust_toolchain = rule( "rustc_flags": attrs.list(attrs.string(), default = []), "rustc_target_triple": attrs.string(default = "x86_64-unknown-linux-gnu"), "rustc_test_flags": attrs.list(attrs.string(), default = []), + "rustdoc": attrs.string(default = "rustdoc"), "rustdoc_flags": attrs.list(attrs.string(), default = []), "warn_lints": attrs.list(attrs.string(), default = []), }, From 51307008d9e53612c52f73d6a013566da23b1f11 Mon Sep 17 00:00:00 2001 From: Nicolas Lopez Date: Thu, 19 Dec 2024 11:21:08 -0500 Subject: [PATCH 09/14] update readme --- buck2/rust/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/buck2/rust/README.md b/buck2/rust/README.md index e03ea0c2..a5b1184f 100644 --- a/buck2/rust/README.md +++ b/buck2/rust/README.md @@ -11,13 +11,13 @@ It is based on two existing samples in the Buck2 upstream repo: In the `platforms` cell we specify: In the `platforms` cell we specify: -* The platform used for remote execution in this project `root//platforms:remote_platform`, which includes the definition of the Docker image used for remote execution, and that defines constraints for targets to run in the remote execution environment. This platform provides an `ExecutionPlatformRegistrationInfo` a `ConfigurationInfo` and a `PlatformInfo` to be able to be used in the `.buckconfig`, in the `--target-platforms` flag, and in the `exec_compatible_with` of `rust_*` rules. Note since rust rules depend on C++ linking this is needed similarly to the Buck2 cpp example in this repo. +* The platform used for remote execution in this project `root//platforms:remote_platform`, which includes the definition of the Docker image used for remote execution, and that defines constraints for targets to run in the remote execution environment. This platform provides an `ExecutionPlatformRegistrationInfo` a `ConfigurationInfo` and a `PlatformInfo` to be able to be used in the `.buckconfig`, and in the `--target-platforms` flag. * The action keys `root//platforms:remote_execution_action_keys`, which provides a default `BuildModeInfo` that is needed for RE of tests to function properly. * The platform `image` configured in `platforms/defs.bzl`, notably, uses a different image than other Buck2 samples in this repo. Specifically, it uses a public AWS image that has `rust` preinstalled. This is because Buck2 rust rules do not include a hermetic rust toolchain. In the `toolchains` cell we specify: -* The remote rust toolchain `root//toolchains:remote_rust_toolchain` which is compatible with the remote execution environment. This toolchain is configured with the `rustc_target_triple` to match the remote execution environment. +* The remote rust toolchain `root//toolchains:remote_rust_toolchain` which is compatible with the remote execution environment. This toolchain is configured with the `rustc_target_triple` to match the remote execution environment, and the compiler, clippy * The c++ toolchain `root//toolchains:cxx_tools_info_toolchain` that is compatible with the remote execution environment. * The clang tools, `root//toolchains:path_clang_tools`, which is used by the c++ toolchain, and specifies the tools installed in the Docker image. * The remote test execution toolchain, `root//toolchains:remote_test_execution_toolchain`. This toolchain defines platform options in the form of `capabilities`. Critically these include the `container-image`. This toolchain is identical to the one in the `buck2/cpp` sample in this repo. From 6dbe77bc6542e09de74878b2d6a20112e693599f Mon Sep 17 00:00:00 2001 From: Nicolas Lopez Date: Thu, 19 Dec 2024 11:22:29 -0500 Subject: [PATCH 10/14] update readme --- buck2/rust/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/buck2/rust/README.md b/buck2/rust/README.md index a5b1184f..0ff2ac0f 100644 --- a/buck2/rust/README.md +++ b/buck2/rust/README.md @@ -13,11 +13,11 @@ In the `platforms` cell we specify: In the `platforms` cell we specify: * The platform used for remote execution in this project `root//platforms:remote_platform`, which includes the definition of the Docker image used for remote execution, and that defines constraints for targets to run in the remote execution environment. This platform provides an `ExecutionPlatformRegistrationInfo` a `ConfigurationInfo` and a `PlatformInfo` to be able to be used in the `.buckconfig`, and in the `--target-platforms` flag. * The action keys `root//platforms:remote_execution_action_keys`, which provides a default `BuildModeInfo` that is needed for RE of tests to function properly. -* The platform `image` configured in `platforms/defs.bzl`, notably, uses a different image than other Buck2 samples in this repo. Specifically, it uses a public AWS image that has `rust` preinstalled. This is because Buck2 rust rules do not include a hermetic rust toolchain. +* The platform `image` configured in `platforms/defs.bzl`, notably, uses a different image than other Buck2 samples in this repo. Specifically, it uses a public AWS image that has `rust` preinstalled. This is because Buck2 rust rules do not include a hermetic rust toolchain. We use a bookworm image with rust pre-installed. Image details can be found in https://gallery.ecr.aws/docker/library/rust. The Dockerfile can be found in https://github.com/rust-lang/docker-rust/blob/700c4f146427808cfb1e07a646e4afabbe99da4f/stable/bullseye/Dockerfile. If a different version of `rust` or other tools is needed you should create your own image and publish it to a repo that is accessible from the cluster. In the `toolchains` cell we specify: -* The remote rust toolchain `root//toolchains:remote_rust_toolchain` which is compatible with the remote execution environment. This toolchain is configured with the `rustc_target_triple` to match the remote execution environment, and the compiler, clippy +* The remote rust toolchain `root//toolchains:remote_rust_toolchain` which is compatible with the remote execution environment. This toolchain is configured with the `rustc_target_triple` to match the remote execution environment, and the `compiler`, `clippy_driver` and `rustdoc` attrs are set to point to the location of the `rustc` binary in the image used for remote execution. * The c++ toolchain `root//toolchains:cxx_tools_info_toolchain` that is compatible with the remote execution environment. * The clang tools, `root//toolchains:path_clang_tools`, which is used by the c++ toolchain, and specifies the tools installed in the Docker image. * The remote test execution toolchain, `root//toolchains:remote_test_execution_toolchain`. This toolchain defines platform options in the form of `capabilities`. Critically these include the `container-image`. This toolchain is identical to the one in the `buck2/cpp` sample in this repo. From 7ffec0cf9c6688993e882a3e319f96c88c5df5e7 Mon Sep 17 00:00:00 2001 From: Nicolas Lopez Date: Tue, 7 Jan 2025 10:04:30 -0500 Subject: [PATCH 11/14] add todo to rust sample --- buck2/rust/BUCK | 1 + 1 file changed, 1 insertion(+) diff --git a/buck2/rust/BUCK b/buck2/rust/BUCK index 3cc3dafa..2542c69f 100644 --- a/buck2/rust/BUCK +++ b/buck2/rust/BUCK @@ -34,6 +34,7 @@ rust_test( ["test/**/*.rs"], ), deps = [":library"], + # TODO: remove these once https://github.com/facebook/buck2/pull/826 gets merged. remote_execution_action_key_providers = select({ "//platforms:engflow": "//platforms:remote_execution_action_keys", "DEFAULT": None, From cdfdb166a5b3476db1a367f75594b182b0a46319 Mon Sep 17 00:00:00 2001 From: Nicolas Lopez Date: Tue, 7 Jan 2025 13:22:36 -0500 Subject: [PATCH 12/14] use --remote_only --- buck2/python/.buckconfig | 2 +- infra/test-buck2.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/buck2/python/.buckconfig b/buck2/python/.buckconfig index 8e773e99..eceb0450 100644 --- a/buck2/python/.buckconfig +++ b/buck2/python/.buckconfig @@ -14,7 +14,7 @@ prelude = bundled [parser] - target_platform_detector_spec = target:root//...->prelude//platforms:default + target_platform_detector_spec = target:root//...->root//platforms:remote_platform [buck2] digest_algorithms = SHA256 diff --git a/infra/test-buck2.sh b/infra/test-buck2.sh index e5ee92b2..6dd778bb 100755 --- a/infra/test-buck2.sh +++ b/infra/test-buck2.sh @@ -24,6 +24,6 @@ cd .. # Run rust example cd rust -buck2 build --target-platforms //platforms:remote_platform //:main -buck2 test --target-platforms //platforms:remote_platform //:test +buck2 build --remote-only //:main +buck2 test --remote-only //:test cd .. From 7a217d56ddb14d3c61dd998753c99fb5651c7dad Mon Sep 17 00:00:00 2001 From: Nicolas Lopez Date: Tue, 7 Jan 2025 13:27:44 -0500 Subject: [PATCH 13/14] fix --- buck2/python/.buckconfig | 2 +- buck2/rust/.buckconfig | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/buck2/python/.buckconfig b/buck2/python/.buckconfig index eceb0450..8e773e99 100644 --- a/buck2/python/.buckconfig +++ b/buck2/python/.buckconfig @@ -14,7 +14,7 @@ prelude = bundled [parser] - target_platform_detector_spec = target:root//...->root//platforms:remote_platform + target_platform_detector_spec = target:root//...->prelude//platforms:default [buck2] digest_algorithms = SHA256 diff --git a/buck2/rust/.buckconfig b/buck2/rust/.buckconfig index 8e773e99..eceb0450 100644 --- a/buck2/rust/.buckconfig +++ b/buck2/rust/.buckconfig @@ -14,7 +14,7 @@ prelude = bundled [parser] - target_platform_detector_spec = target:root//...->prelude//platforms:default + target_platform_detector_spec = target:root//...->root//platforms:remote_platform [buck2] digest_algorithms = SHA256 From 47d3291954bf6a00de51dcb23378cdd59edf755b Mon Sep 17 00:00:00 2001 From: Nicolas Lopez Date: Wed, 8 Jan 2025 13:39:59 -0500 Subject: [PATCH 14/14] more review comments --- buck2/rust/.buckconfig | 24 ++++++++++++------------ buck2/rust/README.md | 8 ++++---- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/buck2/rust/.buckconfig b/buck2/rust/.buckconfig index eceb0450..fc5ca33f 100644 --- a/buck2/rust/.buckconfig +++ b/buck2/rust/.buckconfig @@ -1,20 +1,20 @@ [cells] - root = . - prelude = prelude - toolchains = toolchains - none = none +root = . +prelude = prelude +toolchains = toolchains +none = none [cell_aliases] - config = prelude - fbcode = none - fbsource = none - buck = none +config = prelude +fbcode = none +fbsource = none +buck = none [external_cells] - prelude = bundled +prelude = bundled [parser] - target_platform_detector_spec = target:root//...->root//platforms:remote_platform +target_platform_detector_spec = target:root//...->root//platforms:remote_platform [buck2] digest_algorithms = SHA256 @@ -26,7 +26,7 @@ cas_address = .cluster.engflow.com http_headers = [build] - execution_platforms = root//platforms:remote_platform +execution_platforms = root//platforms:remote_platform [project] - ignore = .git +ignore = .git diff --git a/buck2/rust/README.md b/buck2/rust/README.md index 0ff2ac0f..8eefd092 100644 --- a/buck2/rust/README.md +++ b/buck2/rust/README.md @@ -11,7 +11,7 @@ It is based on two existing samples in the Buck2 upstream repo: In the `platforms` cell we specify: In the `platforms` cell we specify: -* The platform used for remote execution in this project `root//platforms:remote_platform`, which includes the definition of the Docker image used for remote execution, and that defines constraints for targets to run in the remote execution environment. This platform provides an `ExecutionPlatformRegistrationInfo` a `ConfigurationInfo` and a `PlatformInfo` to be able to be used in the `.buckconfig`, and in the `--target-platforms` flag. +* The platform used for remote execution in this project `root//platforms:remote_platform`, which includes the definition of the Docker image used for remote execution, and that defines constraints for targets to run in the remote execution environment. This platform provides an `ExecutionPlatformRegistrationInfo` a `ConfigurationInfo` and a `PlatformInfo` to be able to be used in the `.buckconfig`. * The action keys `root//platforms:remote_execution_action_keys`, which provides a default `BuildModeInfo` that is needed for RE of tests to function properly. * The platform `image` configured in `platforms/defs.bzl`, notably, uses a different image than other Buck2 samples in this repo. Specifically, it uses a public AWS image that has `rust` preinstalled. This is because Buck2 rust rules do not include a hermetic rust toolchain. We use a bookworm image with rust pre-installed. Image details can be found in https://gallery.ecr.aws/docker/library/rust. The Dockerfile can be found in https://github.com/rust-lang/docker-rust/blob/700c4f146427808cfb1e07a646e4afabbe99da4f/stable/bullseye/Dockerfile. If a different version of `rust` or other tools is needed you should create your own image and publish it to a repo that is accessible from the cluster. @@ -31,16 +31,16 @@ The `src`, `bin` and `test` cells: To test the project with RE run (after setting up `.buckconfig` as indicated below): ``` -buck2 test --target-platforms //platforms:remote_platform //:test +buck2 test --remote-only //:test ``` You can also build the `main` for this sample by running: ``` -buck2 build --target-platforms //platforms:remote_platform //:main +buck2 build --remote-only //:main ``` -Note the use of `--target-platforms` to select the remote platform as part of the build / test command. +Note the use of `--remote-only` to indicate remote execution should be used for the build / test command. ### Relevant configs in `.buckconfig`