From af31412d6f1fe6de18412cf2a2b581c1ff77948c Mon Sep 17 00:00:00 2001 From: Anatolios Laskaris Date: Mon, 5 Jun 2023 18:05:24 +0300 Subject: [PATCH] chore: Prepare for multiplatform build (#148) * Prepare for multiarch * Fix * Fix * Fix * Move some code to scripts * Cleanup * Run chmod * Typo * Fix * Fixes * Typo * Install npm * Fix * Cleanup * Fix * Fix? * Updates --- .github/workflows/update_fluence.yml | 27 ++------ Dockerfile | 77 ++++++++++------------ docker/download_bitcoin_cli.sh | 15 +++++ docker/download_ipfs_fs_repo_migrations.sh | 12 ++++ docker/install_libssl.sh | 14 ++++ fluence/download_fluence.sh | 76 +++++++++++++++------ fluence/fluence.json | 13 +++- s6/minimal/run_fluence | 6 +- 8 files changed, 153 insertions(+), 87 deletions(-) create mode 100755 docker/download_bitcoin_cli.sh create mode 100755 docker/download_ipfs_fs_repo_migrations.sh create mode 100755 docker/install_libssl.sh diff --git a/.github/workflows/update_fluence.yml b/.github/workflows/update_fluence.yml index 051f194..240e6c5 100644 --- a/.github/workflows/update_fluence.yml +++ b/.github/workflows/update_fluence.yml @@ -1,22 +1,16 @@ -name: "update_fluence" +name: "Update nox version" on: - workflow_dispatch: + workflow_call: inputs: version: - description: 'Fluence version' - required: true - url: - description: 'Fluence binary URL' - required: true - sha256: - description: 'Fluence binary SHA256 hash' + description: 'Contents of fluence.json file' required: true jobs: - update_fluence: - name: "Update Fluence" - runs-on: "ubuntu-latest" + update: + name: "Update version" + runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -24,14 +18,7 @@ jobs: token: ${{ secrets.FLUENCEBOT_RELEASE_PLEASE_PAT }} - name: Update Fluence in fluence.json to ${{ github.event.inputs.version }} - run: | - cat < fluence/fluence.json - { - "version": "${{ github.event.inputs.version }}", - "url": "${{ github.event.inputs.url }}", - "sha256": "${{ github.event.inputs.sha256 }}" - } - FLUENCE + run: echo ${{ inputs.version }} | jq > fluence/fluence.json - name: Commit updated fluence.json uses: stefanzweifel/git-auto-commit-action@v4 diff --git a/Dockerfile b/Dockerfile index ab8fecc..3e949d5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,26 +6,29 @@ ARG BITCOIN_CLI_VERSION=23.0 # prepare stage images # ---------------------------------------------------------------------------- -FROM ethereum/client-go:release-${GETH_VERSION} as prepare-geth -FROM ipfs/go-ipfs:v${IPFS_VERSION} as prepare-ipfs +FROM --platform=$TARGETPLATFORM ethereum/client-go:release-${GETH_VERSION} as prepare-geth +FROM --platform=$TARGETPLATFORM ipfs/go-ipfs:v${IPFS_VERSION} as prepare-ipfs -FROM alpine as prepare-bitcoin +FROM --platform=$TARGETPLATFORM alpine as prepare-bitcoin +ARG TARGETPLATFORM +ARG BUILDPLATFORM ARG BITCOIN_CLI_VERSION + # Download checksums ADD https://bitcoincore.org/bin/bitcoin-core-${BITCOIN_CLI_VERSION}/SHA256SUMS ./ -# Download archive -ADD https://bitcoincore.org/bin/bitcoin-core-${BITCOIN_CLI_VERSION}/bitcoin-${BITCOIN_CLI_VERSION}-x86_64-linux-gnu.tar.gz ./ -# Verify that downloaded archive matches exactly the hash that's provided -RUN grep " bitcoin-${BITCOIN_CLI_VERSION}-x86_64-linux-gnu.tar.gz\$" SHA256SUMS | sha256sum -c - -# Extract -RUN tar -xzf "bitcoin-${BITCOIN_CLI_VERSION}-x86_64-linux-gnu.tar.gz" + +# Download bitcoin archive +COPY docker/download_bitcoin_cli.sh /docker/download_bitcoin_cli.sh +RUN /docker/download_bitcoin_cli.sh # minimal # ---------------------------------------------------------------------------- -FROM ghcr.io/linuxserver/baseimage-ubuntu:jammy as minimal +FROM --platform=$TARGETPLATFORM ghcr.io/linuxserver/baseimage-ubuntu:jammy as minimal +ARG TARGETPLATFORM +ARG BUILDPLATFORM # https://github.com/opencontainers/image-spec/blob/main/annotations.md#pre-defined-annotation-keys -LABEL org.opencontainers.image.base.name="ghcr.io/linuxserver/baseimage-ubuntu:focal" +LABEL org.opencontainers.image.base.name="ghcr.io/linuxserver/baseimage-ubuntu:jammy" LABEL org.opencontainers.image.url="https://github.com/fluencelabs/rust-peer-distro" LABEL org.opencontainers.image.vendor="fluencelabs" LABEL maintainer="fluencelabs" @@ -38,26 +41,17 @@ ENV RUST_BACKTRACE="1" ## set /run_fluence as the CMD binary ENV S6_CMD_ARG0="/run_fluence" -RUN \ - echo "**** install packages ****" && \ +RUN --mount=type=cache,target=/var/cache/apt \ apt-get update && \ apt-get install -y --no-install-recommends \ jq \ less \ logrotate \ - curl wget && \ - echo "**** cleanup ****" && \ - apt-get clean && \ - rm -rf \ - /tmp/* \ - /var/lib/apt/lists/* \ - /var/tmp/* + curl wget # install missing libssl -RUN \ - wget http://security.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2_amd64.deb \ - && dpkg -i libssl1.1_1.1.1f-1ubuntu2_amd64.deb \ - && rm libssl1.1_1.1.1f-1ubuntu2_amd64.deb +COPY docker/install_libssl.sh /docker/install_libssl.sh +RUN /docker/install_libssl.sh # aqua-ipfs builtin default env variables # instruct aqua-ipfs (client) to work with an IPFS node hosted on ipfs.fluence.dev @@ -75,9 +69,10 @@ ENV FLUENCE_ENV_CONNECTOR_CONTRACT_ADDRESS=0xb497e025D3095A197E30Ca84DEc36a637E6 # find deals from this block ENV FLUENCE_ENV_CONNECTOR_FROM_BLOCK=0x75f3fbc -# download fluence, builtins -RUN --mount=type=bind,source=fluence,target=/fluence /fluence/download_builtins.sh /fluence/services.json -RUN --mount=type=bind,source=fluence,target=/fluence /fluence/download_fluence.sh /fluence/fluence.json +# download rust-peer binary, builtins +COPY fluence/ /fluence/ +RUN /fluence/download_builtins.sh /fluence/services.json +RUN /fluence/download_fluence.sh /fluence/fluence.json # copy default fluence config COPY fluence/Config.default.toml /.fluence/v1/Config.toml @@ -92,6 +87,8 @@ COPY s6/minimal/ / # ipfs # ---------------------------------------------------------------------------- FROM minimal as ipfs +ARG TARGETPLATFORM +ARG BUILDPLATFORM LABEL org.opencontainers.image.description="rust-peer bundled with IPFS daemon" LABEL dev.fluence.bundles.ipfs="${IPFS_VERSION}" @@ -110,8 +107,9 @@ ENV IPFS_ADDRESSES_ANNOUNCE=/ip4/127.0.0.1/tcp/4001,/ip4/127.0.0.1/tcp/4001/ws ENV FLUENCE_ENV_AQUA_IPFS_EXTERNAL_API_MULTIADDR=/ip4/127.0.0.1/tcp/5001 ENV FLUENCE_ENV_AQUA_IPFS_LOCAL_API_MULTIADDR=/ip4/127.0.0.1/tcp/5001 -# download fs-repo-migrations -RUN wget -qO - "https://dist.ipfs.io/fs-repo-migrations/v2.0.2/fs-repo-migrations_v2.0.2_linux-amd64.tar.gz" | tar -C /usr/local/bin --strip-components=1 -zxvf - +# download ipfs fs-repo-migrations tool +COPY docker/download_ipfs_fs_repo_migrations.sh /docker/download_ipfs_fs_repo_migrations.sh +RUN /docker/download_ipfs_fs_repo_migrations.sh # copy s6 configs COPY s6/ipfs/ / @@ -126,6 +124,8 @@ ARG CERAMIC_VERSION ARG GLAZED_VERSION ARG GETH_VERSION ARG BITCOIN_CLI_VERSION +ARG TARGETPLATFORM +ARG BUILDPLATFORM LABEL org.opencontainers.image.description="rust-peer bundled with IPFS, Ceramic CLI and other tools" LABEL dev.fluence.image.bundles.ceramic="${CERAMIC_VERSION}" @@ -137,24 +137,17 @@ LABEL dev.fluence.image.bundles.geth="${GETH_VERSION}" RUN curl -fsSL https://deb.nodesource.com/gpgkey/nodesource.gpg.key | gpg --dearmor > /usr/share/keyrings/nodesource.gpg \ && echo "deb [signed-by=/usr/share/keyrings/nodesource.gpg] https://deb.nodesource.com/node_16.x focal main" > /etc/apt/sources.list.d/nodesource.list -RUN \ - echo "**** install packages ****" && \ +RUN --mount=type=cache,target=/var/cache/apt \ apt-get update && \ apt-get install -y --no-install-recommends \ musl \ - nodejs && \ - echo "**** cleanup ****" && \ - apt-get clean && \ - rm -rf \ - /tmp/* \ - /var/lib/apt/lists/* \ - /var/tmp/* + nodejs # install ceramic and glaze -RUN npm install --cache /cache --global \ - @ceramicnetwork/cli@$CERAMIC_VERSION \ - @glazed/cli@$GLAZED_VERSION \ - && rm -rf /cache +RUN --mount=type=cache,target=/var/cache/npm \ + npm install --cache /var/cache/npm --global \ + @ceramicnetwork/cli@$CERAMIC_VERSION \ + @glazed/cli@$GLAZED_VERSION # copy geth COPY --from=prepare-geth /usr/local/bin/geth /usr/bin/geth diff --git a/docker/download_bitcoin_cli.sh b/docker/download_bitcoin_cli.sh new file mode 100755 index 0000000..6015544 --- /dev/null +++ b/docker/download_bitcoin_cli.sh @@ -0,0 +1,15 @@ +#! /usr/bin/env sh + +case "$TARGETPLATFORM" in +'linux/amd64') + ARCHIVE="bitcoin-${BITCOIN_CLI_VERSION}-x86_64-linux-gnu.tar.gz" + ;; +'linux/arm64') + ARCHIVE="bitcoin-${BITCOIN_CLI_VERSION}-aarch64-linux-gnu.tar.gz" + ;; +esac + +wget "https://bitcoincore.org/bin/bitcoin-core-${BITCOIN_CLI_VERSION}/$ARCHIVE" +grep " $ARCHIVE\$" SHA256SUMS | sha256sum -c - +tar -xzf "$ARCHIVE" +rm "$ARCHIVE" diff --git a/docker/download_ipfs_fs_repo_migrations.sh b/docker/download_ipfs_fs_repo_migrations.sh new file mode 100755 index 0000000..6a2c444 --- /dev/null +++ b/docker/download_ipfs_fs_repo_migrations.sh @@ -0,0 +1,12 @@ +#! /usr/bin/env sh + +case "$TARGETPLATFORM" in +'linux/amd64') + ARCHIVE="fs-repo-migrations_v2.0.2_linux-amd64.tar.gz" + ;; +'linux/arm64') + ARCHIVE="fs-repo-migrations_v2.0.2_linux-arm64.tar.gz" + ;; +esac + +wget -qO - "https://dist.ipfs.io/fs-repo-migrations/v2.0.2/$ARCHIVE" | tar -C /usr/local/bin --strip-components=1 -zxvf - diff --git a/docker/install_libssl.sh b/docker/install_libssl.sh new file mode 100755 index 0000000..0c8dc40 --- /dev/null +++ b/docker/install_libssl.sh @@ -0,0 +1,14 @@ +#! /usr/bin/env sh + +case "$TARGETPLATFORM" in +'linux/amd64') + URL="http://security.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2_amd64.deb" + ;; +'linux/arm64') + URL="http://launchpadlibrarian.net/668086110/libssl1.1_1.1.1-1ubuntu2.1~18.04.23_arm64.deb" + ;; +esac + +wget $URL -O libssl.deb +dpkg -i libssl.deb +rm libssl.deb diff --git a/fluence/download_fluence.sh b/fluence/download_fluence.sh index 4621504..afd5910 100755 --- a/fluence/download_fluence.sh +++ b/fluence/download_fluence.sh @@ -1,24 +1,62 @@ #!/usr/bin/env bash set -o pipefail -o errexit -o nounset +# Validate necessary commands and given config file +for cmd in jq curl sha256sum; do + if ! command -v $cmd &> /dev/null; then + echo "$cmd could not be found, please install it." + exit + fi +done + CONFIG="${1:-fluence.json}" -jq -r '.url, .sha256, .version' $CONFIG | - while - IFS='' - read -r url - read -r sha256 - read -r version - do - echo "*** download $version ***" - # TODO: use --fail-with-body - curl -sL --fail $url -o /usr/bin/fluence || ( - echo "failed to download $url" >&2 - exit 1 - ) - echo "$sha256 /usr/bin/fluence" | sha256sum --check --status || ( - echo "incorrect SHA256" >&2 - exit 1 - ) - chmod +x /usr/bin/fluence - done +if [[ ! -f "$CONFIG" ]]; then + echo "Config file $CONFIG not found!" + exit 1 +fi + +# Detect system architecture +ARCHITECTURE=$(uname -m) + +# Map common architecture names to the ones used in your JSON +case "$ARCHITECTURE" in + x86_64) + KEY="x86_64" + ;; + aarch64) + KEY="aarch64" + ;; + *) + echo "Unsupported architecture: $ARCHITECTURE. Exiting..." + exit 1 + ;; +esac + +# Fetch URL, SHA256, and version based on detected architecture +URL=$(jq -r ".${KEY}.url" $CONFIG) +SHA256=$(jq -r ".${KEY}.sha256" $CONFIG) +VERSION=$(jq -r ".${KEY}.version" $CONFIG) + +echo "*** Downloading rust-peer version $VERSION for $ARCHITECTURE ***" + +ATTEMPTS=5 +while ((ATTEMPTS)); do + curl -sL --fail $URL -o /usr/bin/rust-peer && break + ((ATTEMPTS--)) + sleep 5 +done + +if ! ((ATTEMPTS)); then + echo "Failed to download $URL after 5 attempts. Exiting..." + exit 1 +fi + +if ! echo "$SHA256 /usr/bin/rust-peer" | sha256sum --check --status; then + echo "Incorrect SHA256 for the downloaded file. Exiting..." + exit 1 +fi + +chmod +x /usr/bin/rust-peer + +echo "*** Successfully installed rust-peer version $VERSION for $ARCHITECTURE ***" diff --git a/fluence/fluence.json b/fluence/fluence.json index 65f0055..ac755c2 100644 --- a/fluence/fluence.json +++ b/fluence/fluence.json @@ -1,5 +1,12 @@ { - "version": "0.12.1", - "url": "https://github.com/fluencelabs/rust-peer/releases/download/rust-peer-v0.12.1/rust-peer", - "sha256": "b17037511a020c28657f274f383ab6ead05da0aa5bc3eabbd60267ddf50a145d" + "x86_64": { + "version": "0.12.1", + "url": "https://github.com/fluencelabs/rust-peer/releases/download/rust-peer-v0.12.1/rust-peer", + "sha256": "b17037511a020c28657f274f383ab6ead05da0aa5bc3eabbd60267ddf50a145d" + }, + "aarch64": { + "version": "0.12.1", + "url": "https://github.com/fluencelabs/rust-peer/releases/download/rust-peer-v0.12.1/rust-peer", + "sha256": "b17037511a020c28657f274f383ab6ead05da0aa5bc3eabbd60267ddf50a145d" + } } diff --git a/s6/minimal/run_fluence b/s6/minimal/run_fluence index 2a9429c..bfb707b 100755 --- a/s6/minimal/run_fluence +++ b/s6/minimal/run_fluence @@ -5,7 +5,7 @@ with-contenv # remove old HOME variable from the environment, so the daemon isn't confused by it -# see https://github.com/fluencelabs/node-distro/issues/14 for more details +# see https://github.com/fluencelabs/nox-distro/issues/14 for more details unexport HOME -# 'setuidgid abc' runs '/usr/bin/fluence' as user 'abc' -s6-setuidgid abc /usr/bin/fluence $@ +# 'setuidgid abc' runs '/usr/bin/nox' as user 'abc' +s6-setuidgid abc /usr/bin/nox $@