chore: Prepare for multiplatform build (#148)

* Prepare for multiarch * Fix * Fix * Fix * Move some code to scripts * Cleanup * Run chmod * Typo * Fix * Fixes * Typo * Install npm * Fix * Cleanup * Fix * Fix? * Updates
2024-12-04 15:20:17 +00:00 · 2023-06-05 18:05:24 +03:00 · 2023-06-05 18:05:24 +03:00 · af31412d6f
commit af31412d6f
parent 3c6a1480e5
8 changed files with 153 additions and 87 deletions
--- a/.github/workflows/update_fluence.yml
+++ b/.github/workflows/update_fluence.yml
@ -1,22 +1,16 @@
-name: "update_fluence"
+name: "Update nox version"

 on:
-  workflow_dispatch:
+  workflow_call:
    inputs:
      version:
-        description: 'Fluence version'
-        required: true
-      url:
-        description: 'Fluence binary URL'
-        required: true
-      sha256:
-        description: 'Fluence binary SHA256 hash'
+        description: 'Contents of fluence.json file'
        required: true

 jobs:
-  update_fluence:
-    name: "Update Fluence"
-    runs-on: "ubuntu-latest"
+  update:
+    name: "Update version"
+    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v3
@ -24,14 +18,7 @@ jobs:
          token: ${{ secrets.FLUENCEBOT_RELEASE_PLEASE_PAT }}

      - name: Update Fluence in fluence.json to ${{ github.event.inputs.version }}
-        run: |
-          cat <<FLUENCE > fluence/fluence.json
-          {
-            "version": "${{ github.event.inputs.version }}",
-            "url": "${{ github.event.inputs.url }}",
-            "sha256": "${{ github.event.inputs.sha256 }}"
-          }
-          FLUENCE
+        run: echo ${{ inputs.version }} | jq > fluence/fluence.json

      - name: Commit updated fluence.json
        uses: stefanzweifel/git-auto-commit-action@v4
--- a/75
+++ b/75
@ -6,26 +6,29 @@ ARG BITCOIN_CLI_VERSION=23.0

 # prepare stage images
 # ----------------------------------------------------------------------------
-FROM ethereum/client-go:release-${GETH_VERSION} as prepare-geth
-FROM ipfs/go-ipfs:v${IPFS_VERSION} as prepare-ipfs
+FROM --platform=$TARGETPLATFORM ethereum/client-go:release-${GETH_VERSION} as prepare-geth
+FROM --platform=$TARGETPLATFORM ipfs/go-ipfs:v${IPFS_VERSION} as prepare-ipfs

-FROM alpine as prepare-bitcoin
+FROM --platform=$TARGETPLATFORM alpine as prepare-bitcoin
+ARG TARGETPLATFORM
+ARG BUILDPLATFORM
 ARG BITCOIN_CLI_VERSION
+
 # Download checksums
 ADD https://bitcoincore.org/bin/bitcoin-core-${BITCOIN_CLI_VERSION}/SHA256SUMS ./
-# Download archive
-ADD https://bitcoincore.org/bin/bitcoin-core-${BITCOIN_CLI_VERSION}/bitcoin-${BITCOIN_CLI_VERSION}-x86_64-linux-gnu.tar.gz ./
-# Verify that downloaded archive matches exactly the hash that's provided
-RUN grep " bitcoin-${BITCOIN_CLI_VERSION}-x86_64-linux-gnu.tar.gz\$" SHA256SUMS | sha256sum -c -
-# Extract
-RUN tar -xzf "bitcoin-${BITCOIN_CLI_VERSION}-x86_64-linux-gnu.tar.gz"
+
+# Download bitcoin archive
+COPY docker/download_bitcoin_cli.sh /docker/download_bitcoin_cli.sh
+RUN /docker/download_bitcoin_cli.sh

 # minimal
 # ----------------------------------------------------------------------------
-FROM ghcr.io/linuxserver/baseimage-ubuntu:jammy as minimal
+FROM --platform=$TARGETPLATFORM ghcr.io/linuxserver/baseimage-ubuntu:jammy as minimal
+ARG TARGETPLATFORM
+ARG BUILDPLATFORM

 # https://github.com/opencontainers/image-spec/blob/main/annotations.md#pre-defined-annotation-keys
-LABEL org.opencontainers.image.base.name="ghcr.io/linuxserver/baseimage-ubuntu:focal"
+LABEL org.opencontainers.image.base.name="ghcr.io/linuxserver/baseimage-ubuntu:jammy"
 LABEL org.opencontainers.image.url="https://github.com/fluencelabs/rust-peer-distro"
 LABEL org.opencontainers.image.vendor="fluencelabs"
 LABEL maintainer="fluencelabs"
@ -38,26 +41,17 @@ ENV RUST_BACKTRACE="1"
 ## set /run_fluence as the CMD binary
 ENV S6_CMD_ARG0="/run_fluence"

-RUN \
-  echo "**** install packages ****" && \
+RUN --mount=type=cache,target=/var/cache/apt \
  apt-get update && \
  apt-get install -y --no-install-recommends \
  	jq \
  	less \
  	logrotate \
-  	curl wget && \
-  echo "**** cleanup ****" && \
-  apt-get clean && \
-  rm -rf \
-  	/tmp/* \
-  	/var/lib/apt/lists/* \
-  	/var/tmp/*
+  	curl wget

 # install missing libssl
-RUN \
-  wget http://security.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2_amd64.deb \
-  && dpkg -i libssl1.1_1.1.1f-1ubuntu2_amd64.deb \
-  && rm libssl1.1_1.1.1f-1ubuntu2_amd64.deb
+COPY docker/install_libssl.sh /docker/install_libssl.sh
+RUN /docker/install_libssl.sh

 # aqua-ipfs builtin default env variables
 # instruct aqua-ipfs (client) to work with an IPFS node hosted on ipfs.fluence.dev
@ -75,9 +69,10 @@ ENV FLUENCE_ENV_CONNECTOR_CONTRACT_ADDRESS=0xb497e025D3095A197E30Ca84DEc36a637E6
 # find deals from this block
 ENV FLUENCE_ENV_CONNECTOR_FROM_BLOCK=0x75f3fbc

-# download fluence, builtins
-RUN --mount=type=bind,source=fluence,target=/fluence /fluence/download_builtins.sh /fluence/services.json
-RUN --mount=type=bind,source=fluence,target=/fluence /fluence/download_fluence.sh /fluence/fluence.json
+# download rust-peer binary, builtins
+COPY fluence/ /fluence/
+RUN /fluence/download_builtins.sh /fluence/services.json
+RUN /fluence/download_fluence.sh /fluence/fluence.json

 # copy default fluence config
 COPY fluence/Config.default.toml /.fluence/v1/Config.toml
@ -92,6 +87,8 @@ COPY s6/minimal/ /
 # ipfs
 # ----------------------------------------------------------------------------
 FROM minimal as ipfs
+ARG TARGETPLATFORM
+ARG BUILDPLATFORM

 LABEL org.opencontainers.image.description="rust-peer bundled with IPFS daemon"
 LABEL dev.fluence.bundles.ipfs="${IPFS_VERSION}"
@ -110,8 +107,9 @@ ENV IPFS_ADDRESSES_ANNOUNCE=/ip4/127.0.0.1/tcp/4001,/ip4/127.0.0.1/tcp/4001/ws
 ENV FLUENCE_ENV_AQUA_IPFS_EXTERNAL_API_MULTIADDR=/ip4/127.0.0.1/tcp/5001
 ENV FLUENCE_ENV_AQUA_IPFS_LOCAL_API_MULTIADDR=/ip4/127.0.0.1/tcp/5001

-# download fs-repo-migrations
-RUN wget -qO - "https://dist.ipfs.io/fs-repo-migrations/v2.0.2/fs-repo-migrations_v2.0.2_linux-amd64.tar.gz" | tar -C /usr/local/bin --strip-components=1 -zxvf -
+# download ipfs fs-repo-migrations tool
+COPY docker/download_ipfs_fs_repo_migrations.sh /docker/download_ipfs_fs_repo_migrations.sh
+RUN /docker/download_ipfs_fs_repo_migrations.sh

 # copy s6 configs
 COPY s6/ipfs/ /
@ -126,6 +124,8 @@ ARG CERAMIC_VERSION
 ARG GLAZED_VERSION
 ARG GETH_VERSION
 ARG BITCOIN_CLI_VERSION
+ARG TARGETPLATFORM
+ARG BUILDPLATFORM

 LABEL org.opencontainers.image.description="rust-peer bundled with IPFS, Ceramic CLI and other tools"
 LABEL dev.fluence.image.bundles.ceramic="${CERAMIC_VERSION}"
@ -137,24 +137,17 @@ LABEL dev.fluence.image.bundles.geth="${GETH_VERSION}"
 RUN curl -fsSL https://deb.nodesource.com/gpgkey/nodesource.gpg.key | gpg --dearmor > /usr/share/keyrings/nodesource.gpg \
    && echo "deb [signed-by=/usr/share/keyrings/nodesource.gpg] https://deb.nodesource.com/node_16.x focal main" > /etc/apt/sources.list.d/nodesource.list

-RUN \
-  echo "**** install packages ****" && \
+RUN --mount=type=cache,target=/var/cache/apt \
  apt-get update && \
  apt-get install -y --no-install-recommends \
    musl \
-    nodejs && \
-  echo "**** cleanup ****" && \
-  apt-get clean && \
-  rm -rf \
-    /tmp/* \
-    /var/lib/apt/lists/* \
-    /var/tmp/*
+    nodejs

 # install ceramic and glaze
-RUN npm install --cache /cache --global \
+RUN --mount=type=cache,target=/var/cache/npm \
+  npm install --cache /var/cache/npm --global \
    @ceramicnetwork/cli@$CERAMIC_VERSION \
-  @glazed/cli@$GLAZED_VERSION \
-  && rm -rf /cache
+    @glazed/cli@$GLAZED_VERSION

 # copy geth
 COPY --from=prepare-geth /usr/local/bin/geth /usr/bin/geth
--- a/docker/download_bitcoin_cli.sh
+++ b/docker/download_bitcoin_cli.sh
@ -0,0 +1,15 @@
+#! /usr/bin/env sh
+
+case "$TARGETPLATFORM" in
+'linux/amd64')
+  ARCHIVE="bitcoin-${BITCOIN_CLI_VERSION}-x86_64-linux-gnu.tar.gz"
+  ;;
+'linux/arm64')
+  ARCHIVE="bitcoin-${BITCOIN_CLI_VERSION}-aarch64-linux-gnu.tar.gz"
+  ;;
+esac
+
+wget "https://bitcoincore.org/bin/bitcoin-core-${BITCOIN_CLI_VERSION}/$ARCHIVE"
+grep " $ARCHIVE\$" SHA256SUMS | sha256sum -c -
+tar -xzf "$ARCHIVE"
+rm "$ARCHIVE"
--- a/docker/download_ipfs_fs_repo_migrations.sh
+++ b/docker/download_ipfs_fs_repo_migrations.sh
@ -0,0 +1,12 @@
+#! /usr/bin/env sh
+
+case "$TARGETPLATFORM" in
+'linux/amd64')
+  ARCHIVE="fs-repo-migrations_v2.0.2_linux-amd64.tar.gz"
+  ;;
+'linux/arm64')
+  ARCHIVE="fs-repo-migrations_v2.0.2_linux-arm64.tar.gz"
+  ;;
+esac
+
+wget -qO - "https://dist.ipfs.io/fs-repo-migrations/v2.0.2/$ARCHIVE" | tar -C /usr/local/bin --strip-components=1 -zxvf -
--- a/docker/install_libssl.sh
+++ b/docker/install_libssl.sh
@ -0,0 +1,14 @@
+#! /usr/bin/env sh
+
+case "$TARGETPLATFORM" in
+'linux/amd64')
+  URL="http://security.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2_amd64.deb"
+  ;;
+'linux/arm64')
+  URL="http://launchpadlibrarian.net/668086110/libssl1.1_1.1.1-1ubuntu2.1~18.04.23_arm64.deb"
+  ;;
+esac
+
+wget $URL -O libssl.deb
+dpkg -i libssl.deb
+rm libssl.deb
--- a/fluence/download_fluence.sh
+++ b/fluence/download_fluence.sh
@ -1,24 +1,62 @@
 #!/usr/bin/env bash
 set -o pipefail -o errexit -o nounset

+# Validate necessary commands and given config file
+for cmd in jq curl sha256sum; do
+    if ! command -v $cmd &> /dev/null; then
+        echo "$cmd could not be found, please install it."
+        exit
+    fi
+done
+
 CONFIG="${1:-fluence.json}"

-jq -r '.url, .sha256, .version' $CONFIG |
-    while
-        IFS=''
-        read -r url
-        read -r sha256
-        read -r version
-    do
-        echo "*** download $version ***"
-        # TODO: use --fail-with-body
-        curl -sL --fail $url -o /usr/bin/fluence || (
-            echo "failed to download $url" >&2
+if [[ ! -f "$CONFIG" ]]; then
+    echo "Config file $CONFIG not found!"
    exit 1
-        )
-        echo "$sha256 /usr/bin/fluence" | sha256sum --check --status || (
-            echo "incorrect SHA256" >&2
+fi
+
+# Detect system architecture
+ARCHITECTURE=$(uname -m)
+
+# Map common architecture names to the ones used in your JSON
+case "$ARCHITECTURE" in
+    x86_64)
+        KEY="x86_64"
+        ;;
+    aarch64)
+        KEY="aarch64"
+        ;;
+    *)
+        echo "Unsupported architecture: $ARCHITECTURE. Exiting..."
        exit 1
-        )
-        chmod +x /usr/bin/fluence
+        ;;
+esac
+
+# Fetch URL, SHA256, and version based on detected architecture
+URL=$(jq -r ".${KEY}.url" $CONFIG)
+SHA256=$(jq -r ".${KEY}.sha256" $CONFIG)
+VERSION=$(jq -r ".${KEY}.version" $CONFIG)
+
+echo "*** Downloading rust-peer version $VERSION for $ARCHITECTURE ***"
+
+ATTEMPTS=5
+while ((ATTEMPTS)); do
+    curl -sL --fail $URL -o /usr/bin/rust-peer && break
+    ((ATTEMPTS--))
+    sleep 5
 done
+
+if ! ((ATTEMPTS)); then
+    echo "Failed to download $URL after 5 attempts. Exiting..."
+    exit 1
+fi
+
+if ! echo "$SHA256 /usr/bin/rust-peer" | sha256sum --check --status; then
+    echo "Incorrect SHA256 for the downloaded file. Exiting..."
+    exit 1
+fi
+
+chmod +x /usr/bin/rust-peer
+
+echo "*** Successfully installed rust-peer version $VERSION for $ARCHITECTURE ***"
--- a/fluence/fluence.json
+++ b/fluence/fluence.json
@ -1,5 +1,12 @@
 {
+  "x86_64": {
+    "version": "0.12.1",
+    "url": "https://github.com/fluencelabs/rust-peer/releases/download/rust-peer-v0.12.1/rust-peer",
+    "sha256": "b17037511a020c28657f274f383ab6ead05da0aa5bc3eabbd60267ddf50a145d"
+  },
+  "aarch64": {
    "version": "0.12.1",
    "url": "https://github.com/fluencelabs/rust-peer/releases/download/rust-peer-v0.12.1/rust-peer",
    "sha256": "b17037511a020c28657f274f383ab6ead05da0aa5bc3eabbd60267ddf50a145d"
  }
+}
--- a/s6/minimal/run_fluence
+++ b/s6/minimal/run_fluence
@ -5,7 +5,7 @@
 with-contenv

 # remove old HOME variable from the environment, so the daemon isn't confused by it
-# see https://github.com/fluencelabs/node-distro/issues/14 for more details
+# see https://github.com/fluencelabs/nox-distro/issues/14 for more details
 unexport HOME
-# 'setuidgid abc' runs '/usr/bin/fluence' as user 'abc'
-s6-setuidgid abc /usr/bin/fluence $@
+# 'setuidgid abc' runs '/usr/bin/nox' as user 'abc'
+s6-setuidgid abc /usr/bin/nox $@