diff --git a/nix/.dockerignore b/nix/.dockerignore new file mode 100644 index 00000000..85a8077a --- /dev/null +++ b/nix/.dockerignore @@ -0,0 +1,4 @@ +.venv +docker-bake.hcl +docker-bake.override.hcl +docker-compose.override.yml \ No newline at end of file diff --git a/nix/.gitignore b/nix/.gitignore new file mode 100644 index 00000000..4083cc55 --- /dev/null +++ b/nix/.gitignore @@ -0,0 +1,2 @@ +.venv +.DS_Store \ No newline at end of file diff --git a/nix/Dockerfile b/nix/Dockerfile new file mode 100644 index 00000000..b5c61bf0 --- /dev/null +++ b/nix/Dockerfile @@ -0,0 +1,75 @@ +# syntax=docker.io/docker/dockerfile:1.4 + +# build stage: includes resources necessary for installing dependencies +FROM --platform=linux/riscv64 cartesi/python:3.10-slim-jammy as build-stage +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + build-essential=12.9ubuntu3 \ + && rm -rf /var/apt/lists/* + +RUN python -m venv /opt/venv +ENV PATH="/opt/venv/bin:$PATH" + +COPY requirements.txt . + +RUN pip install -r requirements.txt + +# runtime stage: produces final image that will be executed +FROM --platform=linux/riscv64 cartesi/python:3.10-slim-jammy + +COPY --from=build-stage /opt/venv /opt/venv + +# Download Nix and install it into the system. +RUN apt-get update && apt-get install -y xz-utils curl +COPY nix-tarball.tar.xz ./nix-tarball.tar.xz +RUN tar -xf nix-tarball.tar.xz +RUN addgroup --gid 30000 --system nixbld \ + && for i in $(seq 1 30); do adduser --system --disabled-password --home /var/empty --gecos "Nix build user $i" --uid $((30000 + i)) --ingroup nixbld nixbld$i ; done \ + && mkdir -m 0755 /etc/nix \ + && echo 'sandbox = true' > /etc/nix/nix.conf \ + && echo "build-users-group =" >> /etc/nix/nix.conf \ + && echo "extra-experimental-features = nix-command flakes" >> /etc/nix/nix.conf +# https://github.com/hercules-ci/hercules-ci-agent/issues/183 +RUN mkdir -m 0755 /nix && USER=root sh nix-tarball/install --no-daemon \ + && ln -s /nix/var/nix/profiles/default/etc/profile.d/nix.sh /etc/profile.d/ \ + && rm -rf /var/cache/apk/* \ + && /nix/var/nix/profiles/default/bin/nix-collect-garbage --delete-old \ + && /nix/var/nix/profiles/default/bin/nix-store --optimise \ + && /nix/var/nix/profiles/default/bin/nix-store --verify --check-contents +RUN rm nix-tarball.tar.xz \ + && rm -r /nix-tarball + +# FIXME: add nix to the path in the right way? +RUN echo "$PATH:/nix/store/2qzfvsqb9afhb73cc3yfg8hk2xpxcy47-nix-2.16.0pre20230512_dirty-riscv64-unknown-linux-gnu/bin" > /etc/environment +ENV PATH="${PATH}:/nix/store/2qzfvsqb9afhb73cc3yfg8hk2xpxcy47-nix-2.16.0pre20230512_dirty-riscv64-unknown-linux-gnu/bin" + +RUN chown -R 2874:2874 /nix + +RUN apt-get install -y jq +# TODO remove the flake in favor of a complex non dependency free application +# COPY hello-fo-drvs.txt ./hello-fo-drvs.txt +# RUN nix derivation show -r nixpkgs#hello | jq -r '.[] | select(.outputs.out.hash and .env.urls) | .env.urls' | uniq | sort > ./hello-fo-drvs.txt +# RUN for url in $(cat ./hello-fo-drvs.txt) ; do nix-prefetch-url $url; done + +COPY flake ./flake +RUN chown -R 2874:2874 ./flake +# RUN cd flake && nix build .# -L && nix store delete $(readlink ./result) + +ONBUILD ENV \ + ENV=/etc/profile \ + USER=root \ + PATH=/nix/var/nix/profiles/default/bin:/nix/var/nix/profiles/default/sbin:/bin:/sbin:/usr/bin:/usr/sbin \ + GIT_SSL_CAINFO=/etc/ssl/certs/ca-certificates.crt \ + NIX_SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt + +ENV \ + ENV=/etc/profile \ + USER=root \ + PATH=/nix/var/nix/profiles/default/bin:/nix/var/nix/profiles/default/sbin:/bin:/sbin:/usr/bin:/usr/sbin \ + GIT_SSL_CAINFO=/etc/ssl/certs/ca-certificates.crt \ + NIX_SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt \ + NIX_PATH=/nix/var/nix/profiles/per-user/root/channels + +WORKDIR /opt/cartesi/dapp +COPY ./entrypoint.sh . +COPY ./nix.py . diff --git a/nix/README.md b/nix/README.md new file mode 100644 index 00000000..7b6571a4 --- /dev/null +++ b/nix/README.md @@ -0,0 +1,85 @@ +# Nix Builder Dapp + +This proof of concept allows you to run Nix build inside the Cartesi virtual machine. + +We built Nix tools for riscv64 and installed them on the `cartesi/python:3.10-slim-jammy` image in order to create a reproducible build of a generic software given his depencencies. At the moment the docker image depends on a specific package that we are going to build (GNU hello). + +Since we don't have internet access inside the Cartesi VM we need to provide all the (transitive) dependencies sources, using the nix terminology, we need to provide all the fixed output derivations from the target derivation closure. + +Right now this operation has to be done manually running: + +```shell +nix derivation show -r nixpkgs#hello | jq -r '.[] | select(.outputs.out.hash and .env.urls) | .env.urls' | uniq | sort > fo-drvs`. +``` + +Then the Dockerfile will copy the produced file containing all the dependencies sources + +```shell +for url in $(cat ./hello-fo-drvs.txt) ; do nix-prefetch-url $url; done +``` + +## Build Nix for riscv + +This requires Nix installed with `nix-command` and `flakes` extra experimental features enabled: + +```shell +nix build github:aciceri/nix/riscv64#hydraJobs.binaryTarballCross.x86_64-linux.riscv64-linux -L +``` + +Check `./result` for the tarball containing binaries and installation scripts that will be copied by the `Dockerfile`. + +Currently the `Dockerfile` expects the tarball to be named ` nix-tarball.tar.xz` and contain a folder named `nix-tarball`, so a manually renaming is needed. + + +## Interacting with the application + +Build + +```shell +docker buildx bake --load +``` + +Run + +```shell +docker compose -f ../docker-compose.yml -f ./docker-compose.override.yml up +``` + +Then to ensure that everything is up & running you can use the [frontend-console](../frontend-console) application to interact with the DApp. +Ensure that the [application has already been built](../frontend-console/README.md#building) before using it. + +First, go to a separate terminal window and switch to the `frontend-console` directory: + +```shell +cd frontend-console +``` + +Then, send an input as follows: + +```shell +yarn start input send --payload "message" +yarn run v1.22.5 +$ ts-node src/index.ts input send --payload Nix-loves-Cartesi +connecting to http://localhost:8545 +connected to chain 31337 +using account "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266" +sending "Nix-loves-Cartesi" +transaction: 0x7d65ecd902ffb9279e1f1f4208b2712b766f704249f950277529500057852ed0 +waiting for confirmation... +input 1 added to epoch 0 +✨ Done in 11.15s. +``` + +In order to verify the notices generated by your inputs, run the command: + +```shell +yarn start notice list +yarn run v1.22.5 +$ ts-node src/index.ts notice list +querying http://localhost:4000/graphql for notices of {}... +[{"id":"1","epoch":0,"input":1,"notice":0,"payload":"{\"version\": \"nix (Nix) 2.16.0pre20230512_dirty\\n\", \"path\": \"/nix/store/yx04lw06p3zlkb6sli7ghmq3an9pdqi6-hello-world\\n/nix/store/yx04lw06p3zlkb6sli7ghmq3an9pdqi6-hello-world\\n\", \"content\": \"Hello world!\"}"}] +✨ Done in 3.44s. +``` + +You will be able to get in response the Nix version, the path and the content result of the built app installed on the docker image + diff --git a/nix/docker-bake.hcl b/nix/docker-bake.hcl new file mode 100644 index 00000000..1908f13e --- /dev/null +++ b/nix/docker-bake.hcl @@ -0,0 +1,46 @@ + +group "default" { + targets = ["server", "console"] +} + +target "wrapped" { + context = "../build/docker-riscv" + target = "wrapped-stage" + contexts = { + dapp = "target:dapp" + } +} + +target "fs" { + context = "../build/docker-riscv" + target = "fs-stage" + contexts = { + wrapped = "target:wrapped" + } +} + +target "server" { + context = "../build/docker-riscv" + target = "server-stage" + contexts = { + fs = "target:fs" + } +} + +target "console" { + context = "../build/docker-riscv" + target = "console-stage" + contexts = { + fs = "target:fs" + } +} + +target "machine" { + context = "../build/docker-riscv" + target = "machine-stage" + contexts = { + fs = "target:fs" + } +} + + diff --git a/nix/docker-bake.override.hcl b/nix/docker-bake.override.hcl new file mode 100644 index 00000000..a38a0a41 --- /dev/null +++ b/nix/docker-bake.override.hcl @@ -0,0 +1,23 @@ + +target "dapp" { +} + +variable "TAG" { + default = "devel" +} + +variable "DOCKER_ORGANIZATION" { + default = "cartesi" +} + +target "server" { + tags = ["${DOCKER_ORGANIZATION}/dapp:nix-${TAG}-server"] +} + +target "console" { + tags = ["${DOCKER_ORGANIZATION}/dapp:nix-${TAG}-console"] +} + +target "machine" { + tags = ["${DOCKER_ORGANIZATION}/dapp:nix-${TAG}-machine"] +} diff --git a/nix/docker-compose.override.yml b/nix/docker-compose.override.yml new file mode 100644 index 00000000..d0f62cd8 --- /dev/null +++ b/nix/docker-compose.override.yml @@ -0,0 +1,5 @@ +version: "3" + +services: + server_manager: + image: ${DAPP_IMAGE:-cartesi/dapp:nix-devel-server} diff --git a/nix/entrypoint.sh b/nix/entrypoint.sh new file mode 100755 index 00000000..ad1d5a26 --- /dev/null +++ b/nix/entrypoint.sh @@ -0,0 +1,16 @@ +#!/bin/sh +# Copyright 2022 Cartesi Pte. Ltd. +# +# SPDX-License-Identifier: Apache-2.0 +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. + +set -e +export PATH="/opt/venv/bin:$PATH" +rollup-init python3 nix.py diff --git a/nix/flake/flake.lock b/nix/flake/flake.lock new file mode 100644 index 00000000..c2c44c3a --- /dev/null +++ b/nix/flake/flake.lock @@ -0,0 +1,25 @@ +{ + "nodes": { + "nixpkgs": { + "locked": { + "lastModified": 1683777345, + "narHash": "sha256-V2p/A4RpEGqEZussOnHYMU6XglxBJGCODdzoyvcwig8=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "635a306fc8ede2e34cb3dd0d6d0a5d49362150ed", + "type": "github" + }, + "original": { + "id": "nixpkgs", + "type": "indirect" + } + }, + "root": { + "inputs": { + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/nix/flake/flake.nix b/nix/flake/flake.nix new file mode 100644 index 00000000..7686e1ae --- /dev/null +++ b/nix/flake/flake.nix @@ -0,0 +1,10 @@ +{ + inputs = {}; + outputs = {nixpkgs, ...}: let + pkgs = nixpkgs.legacyPackages.riscv64-linux; + in { + packages.riscv64-linux.default = builtins.toFile + "hello-world" + "Hello world!"; + }; +} \ No newline at end of file diff --git a/nix/flake/result b/nix/flake/result new file mode 120000 index 00000000..a09dc390 --- /dev/null +++ b/nix/flake/result @@ -0,0 +1 @@ +/nix/store/yx04lw06p3zlkb6sli7ghmq3an9pdqi6-hello-world \ No newline at end of file diff --git a/nix/hello-fo-drvs.txt b/nix/hello-fo-drvs.txt new file mode 100644 index 00000000..8562989a --- /dev/null +++ b/nix/hello-fo-drvs.txt @@ -0,0 +1,74 @@ +https://astron.com/pub/file/file-5.44.tar.gz +https://bigsearcher.com/mirrors/gcc/releases/gcc-12.2.0/gcc-12.2.0.tar.xz +https://cdn.kernel.org/pub/linux/kernel/v6.x/linux-6.2.tar.xz +https://cgit.freebsd.org/ports/plain/shells/bash/files/patch-configure?id=3e147a1f594751a68fea00a28090d0792bee0b51 +https://cpan.metacpan.org/src/5.0/perl-5.36.0.tar.gz +https://curl.haxx.se/download/curl-8.0.1.tar.bz2 +https://downloads.sourceforge.net/libisl/isl-0.20.tar.xz +https://downloads.sourceforge.net/project/pcre/pcre/8.45/pcre-8.45.tar.bz2 +https://ftp.gnu.org/gnu/libidn/libidn2-2.3.4.tar.gz +https://ftpmirror.gnu.org/autoconf-archive/autoconf-archive-2022.09.03.tar.xz +https://ftpmirror.gnu.org/autoconf/autoconf-2.71.tar.xz +https://ftpmirror.gnu.org/automake/automake-1.16.5.tar.xz +https://ftpmirror.gnu.org/bash/bash-5.2-patches/bash52-001 +https://ftpmirror.gnu.org/bash/bash-5.2-patches/bash52-002 +https://ftpmirror.gnu.org/bash/bash-5.2-patches/bash52-003 +https://ftpmirror.gnu.org/bash/bash-5.2-patches/bash52-004 +https://ftpmirror.gnu.org/bash/bash-5.2-patches/bash52-005 +https://ftpmirror.gnu.org/bash/bash-5.2-patches/bash52-006 +https://ftpmirror.gnu.org/bash/bash-5.2-patches/bash52-007 +https://ftpmirror.gnu.org/bash/bash-5.2-patches/bash52-008 +https://ftpmirror.gnu.org/bash/bash-5.2-patches/bash52-009 +https://ftpmirror.gnu.org/bash/bash-5.2-patches/bash52-010 +https://ftpmirror.gnu.org/bash/bash-5.2-patches/bash52-011 +https://ftpmirror.gnu.org/bash/bash-5.2-patches/bash52-012 +https://ftpmirror.gnu.org/bash/bash-5.2-patches/bash52-013 +https://ftpmirror.gnu.org/bash/bash-5.2-patches/bash52-014 +https://ftpmirror.gnu.org/bash/bash-5.2-patches/bash52-015 +https://ftpmirror.gnu.org/bash/bash-5.2.tar.gz +https://ftpmirror.gnu.org/binutils/binutils-2.40.tar.bz2 +https://ftpmirror.gnu.org/bison/bison-3.8.2.tar.gz +https://ftpmirror.gnu.org/coreutils/coreutils-9.1.tar.xz +https://ftpmirror.gnu.org/diffutils/diffutils-3.9.tar.xz +https://ftpmirror.gnu.org/ed/ed-1.19.tar.lz +https://ftpmirror.gnu.org/findutils/findutils-4.9.0.tar.xz +https://ftpmirror.gnu.org/gawk/gawk-5.2.1.tar.xz +https://ftpmirror.gnu.org/gettext/gettext-0.21.tar.gz +https://ftpmirror.gnu.org/glibc/glibc-2.37.tar.xz +https://ftpmirror.gnu.org/gmp/gmp-6.2.1.tar.bz2 +https://ftpmirror.gnu.org/grep/grep-3.7.tar.xz +https://ftpmirror.gnu.org/gzip/gzip-1.12.tar.xz +https://ftpmirror.gnu.org/libtool/libtool-2.4.7.tar.gz +https://ftpmirror.gnu.org/libunistring/libunistring-1.1.tar.gz +https://ftpmirror.gnu.org/m4/m4-1.4.19.tar.bz2 +https://ftpmirror.gnu.org/make/make-4.4.1.tar.gz +https://ftpmirror.gnu.org/mpc/mpc-1.3.1.tar.gz +https://ftpmirror.gnu.org/patch/patch-2.7.6.tar.xz +https://ftpmirror.gnu.org/sed/sed-4.9.tar.xz +https://ftpmirror.gnu.org/tar/tar-1.34.tar.xz +https://ftpmirror.gnu.org/texinfo/texinfo-7.0.3.tar.xz +https://ftpmirror.gnu.org/which/which-2.21.tar.gz +https://ftp.suse.com/pub/people/sbrabec/bzip2/for_downstream/bzip2-1.0.6.2-autoconfiscated.patch +https://github.com/besser82/libxcrypt/releases/download/v4.4.33/libxcrypt-4.4.33.tar.xz +https://github.com/libexpat/libexpat/releases/download/R_2_5_0/expat-2.5.0.tar.xz +https://github.com/libffi/libffi/releases/download/v3.4.4/libffi-3.4.4.tar.gz +https://github.com/madler/zlib/releases/download/v1.2.13/zlib-1.2.13.tar.gz +https://github.com/nghttp2/nghttp2/releases/download/v1.51.0/nghttp2-1.51.0.tar.bz2 +https://github.com/NixOS/patchelf/releases/download/0.15.0/patchelf-0.15.0.tar.bz2 +https://git.kernel.org/pub/scm/linux/kernel/git/dhowells/keyutils.git/snapshot/keyutils-1.6.3.tar.gz +https://gitweb.gentoo.org/repo/gentoo.git/plain/sys-apps/file/files/file-5.44-decompress-empty.patch?h=dfc57da515a2aaf085bea68267cc727f1bfaa691 +https://kerberos.org/dist/krb5/1.20/krb5-1.20.1.tar.gz +https://lore.kernel.org/keyrings/20230301134250.301819-1-hi@alyssa.is/raw +https://mirror.easyname.at/nongnu/acl/acl-2.3.1.tar.gz +https://mirror.easyname.at/nongnu/attr/attr-2.5.1.tar.gz +https://mirror.easyname.at/nongnu/lzip/lzip-1.23.tar.gz +https://pkg-config.freedesktop.org/releases/pkg-config-0.29.2.tar.gz +https://sourceware.org/pub/bzip2/bzip2-1.0.8.tar.gz +https://tukaani.org/xz/xz-5.4.2.tar.bz2 +https://www.libssh2.org/download/libssh2-1.10.0.tar.gz +https://www.mpfr.org/mpfr-4.2.0/mpfr-4.2.0.tar.xz +https://www.openssl.org/source/openssl-3.0.8.tar.gz +https://www.python.org/ftp/python/3.10.11/Python-3.10.11.tar.xz +http://tarballs.nixos.org/stdenv-linux/i686/4907fc9e8d0d82b28b3c56e3a478a2882f1d700f/busybox +http://tarballs.nixos.org/stdenv-linux/x86_64/c5aabb0d603e2c1ea05f5a93b3be82437f5ebf31/bootstrap-tools.tar.xz +mirror://gnu/hello/hello-2.12.1.tar.gz diff --git a/nix/nix-tarball.tar.xz b/nix/nix-tarball.tar.xz new file mode 100644 index 00000000..b9198675 Binary files /dev/null and b/nix/nix-tarball.tar.xz differ diff --git a/nix/nix.py b/nix/nix.py new file mode 100644 index 00000000..19dfc706 --- /dev/null +++ b/nix/nix.py @@ -0,0 +1,89 @@ +# Copyright 2022 Cartesi Pte. Ltd. +# +# SPDX-License-Identifier: Apache-2.0 +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. + +from os import environ +import logging +import requests +import subprocess +import json + +def str2hex(str): + return "0x" + str.encode("utf-8").hex() + + +logging.basicConfig(level="INFO") +logger = logging.getLogger(__name__) + +rollup_server = environ["ROLLUP_HTTP_SERVER_URL"] +logger.info(f"HTTP rollup_server url is {rollup_server}") + +def handle_advance(data): + logger.info(f"Received advance request data {data}") + logger.info("Adding notice") + + #FIXME: the path of the nix bin + version = subprocess.check_output("/nix/store/2qzfvsqb9afhb73cc3yfg8hk2xpxcy47-nix-2.16.0pre20230512_dirty-riscv64-unknown-linux-gnu/bin/nix --version", shell=True, stderr=subprocess.STDOUT) + + # Nix does something :) + subprocess.check_output("/nix/store/2qzfvsqb9afhb73cc3yfg8hk2xpxcy47-nix-2.16.0pre20230512_dirty-riscv64-unknown-linux-gnu/bin/nix build /flake", shell=True, stderr=subprocess.STDOUT) + path = subprocess.check_output("readlink result ./result", shell=True, stderr=subprocess.STDOUT) + content = subprocess.check_output("cat ./result", shell=True, stderr=subprocess.STDOUT) + + notice = {"payload": str2hex(json.dumps({ + "version": version.decode(), + "path": path.decode(), + "content": content.decode() + })) + } + + notice_but_is_a_string = (notice) + + logger.info(f"Adding notice log {notice_but_is_a_string}") + + response = requests.post(rollup_server + "/notice", json=notice_but_is_a_string) + logger.info(f"Received notice status {response.status_code} body {response.content}") + + return "accept" + +def handle_inspect(data): + logger.info(f"Received inspect request data {data}") + logger.info("Adding report") + report = {"payload": data["payload"]} + response = requests.post(rollup_server + "/report", json=report) + logger.info(f"Received report status {response.status_code}") + return "accept" + +handlers = { + "advance_state": handle_advance, + "inspect_state": handle_inspect, +} + +finish = {"status": "accept"} +rollup_address = None + +while True: + logger.info("Sending finish") + response = requests.post(rollup_server + "/finish", json=finish) + logger.info(f"Received finish status {response.status_code}") + if response.status_code == 202: + logger.info("No pending rollup request, trying again") + else: + rollup_request = response.json() + data = rollup_request["data"] + if "metadata" in data: + metadata = data["metadata"] + if metadata["epoch_index"] == 0 and metadata["input_index"] == 0: + rollup_address = metadata["msg_sender"] + logger.info(f"Captured rollup address: {rollup_address}") + continue + handler = handlers[rollup_request["request_type"]] + finish["status"] = handler(rollup_request["data"]) diff --git a/nix/requirements.txt b/nix/requirements.txt new file mode 100644 index 00000000..b93e93cc --- /dev/null +++ b/nix/requirements.txt @@ -0,0 +1,2 @@ +requests == 2.23.0 +eth_abi == 3.0.0