chore: Prepare for multiplatform build (#148)

* Prepare for multiarch

* Fix

* Fix

* Fix

* Move some code to scripts

* Cleanup

* Run chmod

* Typo

* Fix

* Fixes

* Typo

* Install npm

* Fix

* Cleanup

* Fix

* Fix?

* Updates

.github/workflows/update_fluence.yml

@@ -1,22 +1,16 @@
-name: "update_fluence"
+name: "Update nox version"
 
 on:
-  workflow_dispatch:
+  workflow_call:
     inputs:
       version:
-        description: 'Fluence version'
+        description: 'Contents of fluence.json file'
-        required: true
-      url:
-        description: 'Fluence binary URL'
-        required: true
-      sha256:
-        description: 'Fluence binary SHA256 hash'
         required: true
 
 jobs:
-  update_fluence:
+  update:
-    name: "Update Fluence"
+    name: "Update version"
-    runs-on: "ubuntu-latest"
+    runs-on: ubuntu-latest
 
     steps:
       - uses: actions/checkout@v3
@@ -24,14 +18,7 @@ jobs:
           token: ${{ secrets.FLUENCEBOT_RELEASE_PLEASE_PAT }}
 
       - name: Update Fluence in fluence.json to ${{ github.event.inputs.version }}
-        run: |
+        run: echo ${{ inputs.version }} | jq > fluence/fluence.json
-          cat <<FLUENCE > fluence/fluence.json
-          {
-            "version": "${{ github.event.inputs.version }}",
-            "url": "${{ github.event.inputs.url }}",
-            "sha256": "${{ github.event.inputs.sha256 }}"
-          }
-          FLUENCE
 
       - name: Commit updated fluence.json
         uses: stefanzweifel/git-auto-commit-action@v4

Dockerfile

@@ -6,26 +6,29 @@ ARG BITCOIN_CLI_VERSION=23.0
 
 # prepare stage images
 # ----------------------------------------------------------------------------
-FROM ethereum/client-go:release-${GETH_VERSION} as prepare-geth
+FROM --platform=$TARGETPLATFORM ethereum/client-go:release-${GETH_VERSION} as prepare-geth
-FROM ipfs/go-ipfs:v${IPFS_VERSION} as prepare-ipfs
+FROM --platform=$TARGETPLATFORM ipfs/go-ipfs:v${IPFS_VERSION} as prepare-ipfs
 
-FROM alpine as prepare-bitcoin
+FROM --platform=$TARGETPLATFORM alpine as prepare-bitcoin
+ARG TARGETPLATFORM
+ARG BUILDPLATFORM
 ARG BITCOIN_CLI_VERSION
+
 # Download checksums
 ADD https://bitcoincore.org/bin/bitcoin-core-${BITCOIN_CLI_VERSION}/SHA256SUMS ./
-# Download archive
+
-ADD https://bitcoincore.org/bin/bitcoin-core-${BITCOIN_CLI_VERSION}/bitcoin-${BITCOIN_CLI_VERSION}-x86_64-linux-gnu.tar.gz ./
+# Download bitcoin archive
-# Verify that downloaded archive matches exactly the hash that's provided
+COPY docker/download_bitcoin_cli.sh /docker/download_bitcoin_cli.sh
-RUN grep " bitcoin-${BITCOIN_CLI_VERSION}-x86_64-linux-gnu.tar.gz\$" SHA256SUMS | sha256sum -c -
+RUN /docker/download_bitcoin_cli.sh
-# Extract
-RUN tar -xzf "bitcoin-${BITCOIN_CLI_VERSION}-x86_64-linux-gnu.tar.gz"
 
 # minimal
 # ----------------------------------------------------------------------------
-FROM ghcr.io/linuxserver/baseimage-ubuntu:jammy as minimal
+FROM --platform=$TARGETPLATFORM ghcr.io/linuxserver/baseimage-ubuntu:jammy as minimal
+ARG TARGETPLATFORM
+ARG BUILDPLATFORM
 
 # https://github.com/opencontainers/image-spec/blob/main/annotations.md#pre-defined-annotation-keys
-LABEL org.opencontainers.image.base.name="ghcr.io/linuxserver/baseimage-ubuntu:focal"
+LABEL org.opencontainers.image.base.name="ghcr.io/linuxserver/baseimage-ubuntu:jammy"
 LABEL org.opencontainers.image.url="https://github.com/fluencelabs/rust-peer-distro"
 LABEL org.opencontainers.image.vendor="fluencelabs"
 LABEL maintainer="fluencelabs"
@@ -38,26 +41,17 @@ ENV RUST_BACKTRACE="1"
 ## set /run_fluence as the CMD binary
 ENV S6_CMD_ARG0="/run_fluence"
 
-RUN \
+RUN --mount=type=cache,target=/var/cache/apt \
-  echo "**** install packages ****" && \
   apt-get update && \
   apt-get install -y --no-install-recommends \
   	jq \
   	less \
   	logrotate \
-  	curl wget && \
+  	curl wget
-  echo "**** cleanup ****" && \
-  apt-get clean && \
-  rm -rf \
-  	/tmp/* \
-  	/var/lib/apt/lists/* \
-  	/var/tmp/*
 
 # install missing libssl
-RUN \
+COPY docker/install_libssl.sh /docker/install_libssl.sh
-  wget http://security.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2_amd64.deb \
+RUN /docker/install_libssl.sh
-  && dpkg -i libssl1.1_1.1.1f-1ubuntu2_amd64.deb \
-  && rm libssl1.1_1.1.1f-1ubuntu2_amd64.deb
 
 # aqua-ipfs builtin default env variables
 # instruct aqua-ipfs (client) to work with an IPFS node hosted on ipfs.fluence.dev
@@ -75,9 +69,10 @@ ENV FLUENCE_ENV_CONNECTOR_CONTRACT_ADDRESS=0xb497e025D3095A197E30Ca84DEc36a637E6
 # find deals from this block
 ENV FLUENCE_ENV_CONNECTOR_FROM_BLOCK=0x75f3fbc
 
-# download fluence, builtins
+# download rust-peer binary, builtins
-RUN --mount=type=bind,source=fluence,target=/fluence /fluence/download_builtins.sh /fluence/services.json
+COPY fluence/ /fluence/
-RUN --mount=type=bind,source=fluence,target=/fluence /fluence/download_fluence.sh /fluence/fluence.json
+RUN /fluence/download_builtins.sh /fluence/services.json
+RUN /fluence/download_fluence.sh /fluence/fluence.json
 
 # copy default fluence config
 COPY fluence/Config.default.toml /.fluence/v1/Config.toml
@@ -92,6 +87,8 @@ COPY s6/minimal/ /
 # ipfs
 # ----------------------------------------------------------------------------
 FROM minimal as ipfs
+ARG TARGETPLATFORM
+ARG BUILDPLATFORM
 
 LABEL org.opencontainers.image.description="rust-peer bundled with IPFS daemon"
 LABEL dev.fluence.bundles.ipfs="${IPFS_VERSION}"
@@ -110,8 +107,9 @@ ENV IPFS_ADDRESSES_ANNOUNCE=/ip4/127.0.0.1/tcp/4001,/ip4/127.0.0.1/tcp/4001/ws
 ENV FLUENCE_ENV_AQUA_IPFS_EXTERNAL_API_MULTIADDR=/ip4/127.0.0.1/tcp/5001
 ENV FLUENCE_ENV_AQUA_IPFS_LOCAL_API_MULTIADDR=/ip4/127.0.0.1/tcp/5001
 
-# download fs-repo-migrations
+# download ipfs fs-repo-migrations tool
-RUN wget -qO - "https://dist.ipfs.io/fs-repo-migrations/v2.0.2/fs-repo-migrations_v2.0.2_linux-amd64.tar.gz" | tar -C /usr/local/bin --strip-components=1 -zxvf -
+COPY docker/download_ipfs_fs_repo_migrations.sh /docker/download_ipfs_fs_repo_migrations.sh
+RUN /docker/download_ipfs_fs_repo_migrations.sh
 
 # copy s6 configs
 COPY s6/ipfs/ /
@@ -126,6 +124,8 @@ ARG CERAMIC_VERSION
 ARG GLAZED_VERSION
 ARG GETH_VERSION
 ARG BITCOIN_CLI_VERSION
+ARG TARGETPLATFORM
+ARG BUILDPLATFORM
 
 LABEL org.opencontainers.image.description="rust-peer bundled with IPFS, Ceramic CLI and other tools"
 LABEL dev.fluence.image.bundles.ceramic="${CERAMIC_VERSION}"
@@ -137,24 +137,17 @@ LABEL dev.fluence.image.bundles.geth="${GETH_VERSION}"
 RUN curl -fsSL https://deb.nodesource.com/gpgkey/nodesource.gpg.key | gpg --dearmor > /usr/share/keyrings/nodesource.gpg \
     && echo "deb [signed-by=/usr/share/keyrings/nodesource.gpg] https://deb.nodesource.com/node_16.x focal main" > /etc/apt/sources.list.d/nodesource.list
 
-RUN \
+RUN --mount=type=cache,target=/var/cache/apt \
-  echo "**** install packages ****" && \
   apt-get update && \
   apt-get install -y --no-install-recommends \
     musl \
-    nodejs && \
+    nodejs
-  echo "**** cleanup ****" && \
-  apt-get clean && \
-  rm -rf \
-    /tmp/* \
-    /var/lib/apt/lists/* \
-    /var/tmp/*
 
 # install ceramic and glaze
-RUN npm install --cache /cache --global \
+RUN --mount=type=cache,target=/var/cache/npm \
-  @ceramicnetwork/cli@$CERAMIC_VERSION \
+  npm install --cache /var/cache/npm --global \
-  @glazed/cli@$GLAZED_VERSION \
+    @ceramicnetwork/cli@$CERAMIC_VERSION \
-  && rm -rf /cache
+    @glazed/cli@$GLAZED_VERSION
 
 # copy geth
 COPY --from=prepare-geth /usr/local/bin/geth /usr/bin/geth

docker/download_bitcoin_cli.sh

@@ -0,0 +1,15 @@
+#! /usr/bin/env sh
+
+case "$TARGETPLATFORM" in
+'linux/amd64')
+  ARCHIVE="bitcoin-${BITCOIN_CLI_VERSION}-x86_64-linux-gnu.tar.gz"
+  ;;
+'linux/arm64')
+  ARCHIVE="bitcoin-${BITCOIN_CLI_VERSION}-aarch64-linux-gnu.tar.gz"
+  ;;
+esac
+
+wget "https://bitcoincore.org/bin/bitcoin-core-${BITCOIN_CLI_VERSION}/$ARCHIVE"
+grep " $ARCHIVE\$" SHA256SUMS | sha256sum -c -
+tar -xzf "$ARCHIVE"
+rm "$ARCHIVE"

docker/download_ipfs_fs_repo_migrations.sh

@@ -0,0 +1,12 @@
+#! /usr/bin/env sh
+
+case "$TARGETPLATFORM" in
+'linux/amd64')
+  ARCHIVE="fs-repo-migrations_v2.0.2_linux-amd64.tar.gz"
+  ;;
+'linux/arm64')
+  ARCHIVE="fs-repo-migrations_v2.0.2_linux-arm64.tar.gz"
+  ;;
+esac
+
+wget -qO - "https://dist.ipfs.io/fs-repo-migrations/v2.0.2/$ARCHIVE" | tar -C /usr/local/bin --strip-components=1 -zxvf -

docker/install_libssl.sh

@@ -0,0 +1,14 @@
+#! /usr/bin/env sh
+
+case "$TARGETPLATFORM" in
+'linux/amd64')
+  URL="http://security.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2_amd64.deb"
+  ;;
+'linux/arm64')
+  URL="http://launchpadlibrarian.net/668086110/libssl1.1_1.1.1-1ubuntu2.1~18.04.23_arm64.deb"
+  ;;
+esac
+
+wget $URL -O libssl.deb
+dpkg -i libssl.deb
+rm libssl.deb

fluence/download_fluence.sh

@@ -1,24 +1,62 @@
 #!/usr/bin/env bash
 set -o pipefail -o errexit -o nounset
 
+# Validate necessary commands and given config file
+for cmd in jq curl sha256sum; do
+    if ! command -v $cmd &> /dev/null; then
+        echo "$cmd could not be found, please install it."
+        exit
+    fi
+done
+
 CONFIG="${1:-fluence.json}"
 
-jq -r '.url, .sha256, .version' $CONFIG |
+if [[ ! -f "$CONFIG" ]]; then
-    while
+    echo "Config file $CONFIG not found!"
-        IFS=''
+    exit 1
-        read -r url
+fi
-        read -r sha256
+
-        read -r version
+# Detect system architecture
-    do
+ARCHITECTURE=$(uname -m)
-        echo "*** download $version ***"
+
-        # TODO: use --fail-with-body
+# Map common architecture names to the ones used in your JSON
-        curl -sL --fail $url -o /usr/bin/fluence || (
+case "$ARCHITECTURE" in
-            echo "failed to download $url" >&2
+    x86_64)
-            exit 1
+        KEY="x86_64"
-        )
+        ;;
-        echo "$sha256 /usr/bin/fluence" | sha256sum --check --status || (
+    aarch64)
-            echo "incorrect SHA256" >&2
+        KEY="aarch64"
-            exit 1
+        ;;
-        )
+    *)
-        chmod +x /usr/bin/fluence
+        echo "Unsupported architecture: $ARCHITECTURE. Exiting..."
-    done
+        exit 1
+        ;;
+esac
+
+# Fetch URL, SHA256, and version based on detected architecture
+URL=$(jq -r ".${KEY}.url" $CONFIG)
+SHA256=$(jq -r ".${KEY}.sha256" $CONFIG)
+VERSION=$(jq -r ".${KEY}.version" $CONFIG)
+
+echo "*** Downloading rust-peer version $VERSION for $ARCHITECTURE ***"
+
+ATTEMPTS=5
+while ((ATTEMPTS)); do
+    curl -sL --fail $URL -o /usr/bin/rust-peer && break
+    ((ATTEMPTS--))
+    sleep 5
+done
+
+if ! ((ATTEMPTS)); then
+    echo "Failed to download $URL after 5 attempts. Exiting..."
+    exit 1
+fi
+
+if ! echo "$SHA256 /usr/bin/rust-peer" | sha256sum --check --status; then
+    echo "Incorrect SHA256 for the downloaded file. Exiting..."
+    exit 1
+fi
+
+chmod +x /usr/bin/rust-peer
+
+echo "*** Successfully installed rust-peer version $VERSION for $ARCHITECTURE ***"

fluence/fluence.json

@@ -1,5 +1,12 @@
 {
-  "version": "0.12.1",
+  "x86_64": {
-  "url": "https://github.com/fluencelabs/rust-peer/releases/download/rust-peer-v0.12.1/rust-peer",
+    "version": "0.12.1",
-  "sha256": "b17037511a020c28657f274f383ab6ead05da0aa5bc3eabbd60267ddf50a145d"
+    "url": "https://github.com/fluencelabs/rust-peer/releases/download/rust-peer-v0.12.1/rust-peer",
+    "sha256": "b17037511a020c28657f274f383ab6ead05da0aa5bc3eabbd60267ddf50a145d"
+  },
+  "aarch64": {
+    "version": "0.12.1",
+    "url": "https://github.com/fluencelabs/rust-peer/releases/download/rust-peer-v0.12.1/rust-peer",
+    "sha256": "b17037511a020c28657f274f383ab6ead05da0aa5bc3eabbd60267ddf50a145d"
+  }
 }

s6/minimal/run_fluence

@@ -5,7 +5,7 @@
 with-contenv
 
 # remove old HOME variable from the environment, so the daemon isn't confused by it
-# see https://github.com/fluencelabs/node-distro/issues/14 for more details
+# see https://github.com/fluencelabs/nox-distro/issues/14 for more details
 unexport HOME
-# 'setuidgid abc' runs '/usr/bin/fluence' as user 'abc'
+# 'setuidgid abc' runs '/usr/bin/nox' as user 'abc'
-s6-setuidgid abc /usr/bin/fluence $@
+s6-setuidgid abc /usr/bin/nox $@