chore: Prepare for multiplatform build (#148)

* Prepare for multiarch

* Fix

* Fix

* Fix

* Move some code to scripts

* Cleanup

* Run chmod

* Typo

* Fix

* Fixes

* Typo

* Install npm

* Fix

* Cleanup

* Fix

* Fix?

* Updates
This commit is contained in:
Anatolios Laskaris 2023-06-05 18:05:24 +03:00 committed by GitHub
parent 3c6a1480e5
commit af31412d6f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 153 additions and 87 deletions

View File

@ -1,22 +1,16 @@
name: "update_fluence" name: "Update nox version"
on: on:
workflow_dispatch: workflow_call:
inputs: inputs:
version: version:
description: 'Fluence version' description: 'Contents of fluence.json file'
required: true
url:
description: 'Fluence binary URL'
required: true
sha256:
description: 'Fluence binary SHA256 hash'
required: true required: true
jobs: jobs:
update_fluence: update:
name: "Update Fluence" name: "Update version"
runs-on: "ubuntu-latest" runs-on: ubuntu-latest
steps: steps:
- uses: actions/checkout@v3 - uses: actions/checkout@v3
@ -24,14 +18,7 @@ jobs:
token: ${{ secrets.FLUENCEBOT_RELEASE_PLEASE_PAT }} token: ${{ secrets.FLUENCEBOT_RELEASE_PLEASE_PAT }}
- name: Update Fluence in fluence.json to ${{ github.event.inputs.version }} - name: Update Fluence in fluence.json to ${{ github.event.inputs.version }}
run: | run: echo ${{ inputs.version }} | jq > fluence/fluence.json
cat <<FLUENCE > fluence/fluence.json
{
"version": "${{ github.event.inputs.version }}",
"url": "${{ github.event.inputs.url }}",
"sha256": "${{ github.event.inputs.sha256 }}"
}
FLUENCE
- name: Commit updated fluence.json - name: Commit updated fluence.json
uses: stefanzweifel/git-auto-commit-action@v4 uses: stefanzweifel/git-auto-commit-action@v4

View File

@ -6,26 +6,29 @@ ARG BITCOIN_CLI_VERSION=23.0
# prepare stage images # prepare stage images
# ---------------------------------------------------------------------------- # ----------------------------------------------------------------------------
FROM ethereum/client-go:release-${GETH_VERSION} as prepare-geth FROM --platform=$TARGETPLATFORM ethereum/client-go:release-${GETH_VERSION} as prepare-geth
FROM ipfs/go-ipfs:v${IPFS_VERSION} as prepare-ipfs FROM --platform=$TARGETPLATFORM ipfs/go-ipfs:v${IPFS_VERSION} as prepare-ipfs
FROM alpine as prepare-bitcoin FROM --platform=$TARGETPLATFORM alpine as prepare-bitcoin
ARG TARGETPLATFORM
ARG BUILDPLATFORM
ARG BITCOIN_CLI_VERSION ARG BITCOIN_CLI_VERSION
# Download checksums # Download checksums
ADD https://bitcoincore.org/bin/bitcoin-core-${BITCOIN_CLI_VERSION}/SHA256SUMS ./ ADD https://bitcoincore.org/bin/bitcoin-core-${BITCOIN_CLI_VERSION}/SHA256SUMS ./
# Download archive
ADD https://bitcoincore.org/bin/bitcoin-core-${BITCOIN_CLI_VERSION}/bitcoin-${BITCOIN_CLI_VERSION}-x86_64-linux-gnu.tar.gz ./ # Download bitcoin archive
# Verify that downloaded archive matches exactly the hash that's provided COPY docker/download_bitcoin_cli.sh /docker/download_bitcoin_cli.sh
RUN grep " bitcoin-${BITCOIN_CLI_VERSION}-x86_64-linux-gnu.tar.gz\$" SHA256SUMS | sha256sum -c - RUN /docker/download_bitcoin_cli.sh
# Extract
RUN tar -xzf "bitcoin-${BITCOIN_CLI_VERSION}-x86_64-linux-gnu.tar.gz"
# minimal # minimal
# ---------------------------------------------------------------------------- # ----------------------------------------------------------------------------
FROM ghcr.io/linuxserver/baseimage-ubuntu:jammy as minimal FROM --platform=$TARGETPLATFORM ghcr.io/linuxserver/baseimage-ubuntu:jammy as minimal
ARG TARGETPLATFORM
ARG BUILDPLATFORM
# https://github.com/opencontainers/image-spec/blob/main/annotations.md#pre-defined-annotation-keys # https://github.com/opencontainers/image-spec/blob/main/annotations.md#pre-defined-annotation-keys
LABEL org.opencontainers.image.base.name="ghcr.io/linuxserver/baseimage-ubuntu:focal" LABEL org.opencontainers.image.base.name="ghcr.io/linuxserver/baseimage-ubuntu:jammy"
LABEL org.opencontainers.image.url="https://github.com/fluencelabs/rust-peer-distro" LABEL org.opencontainers.image.url="https://github.com/fluencelabs/rust-peer-distro"
LABEL org.opencontainers.image.vendor="fluencelabs" LABEL org.opencontainers.image.vendor="fluencelabs"
LABEL maintainer="fluencelabs" LABEL maintainer="fluencelabs"
@ -38,26 +41,17 @@ ENV RUST_BACKTRACE="1"
## set /run_fluence as the CMD binary ## set /run_fluence as the CMD binary
ENV S6_CMD_ARG0="/run_fluence" ENV S6_CMD_ARG0="/run_fluence"
RUN \ RUN --mount=type=cache,target=/var/cache/apt \
echo "**** install packages ****" && \
apt-get update && \ apt-get update && \
apt-get install -y --no-install-recommends \ apt-get install -y --no-install-recommends \
jq \ jq \
less \ less \
logrotate \ logrotate \
curl wget && \ curl wget
echo "**** cleanup ****" && \
apt-get clean && \
rm -rf \
/tmp/* \
/var/lib/apt/lists/* \
/var/tmp/*
# install missing libssl # install missing libssl
RUN \ COPY docker/install_libssl.sh /docker/install_libssl.sh
wget http://security.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2_amd64.deb \ RUN /docker/install_libssl.sh
&& dpkg -i libssl1.1_1.1.1f-1ubuntu2_amd64.deb \
&& rm libssl1.1_1.1.1f-1ubuntu2_amd64.deb
# aqua-ipfs builtin default env variables # aqua-ipfs builtin default env variables
# instruct aqua-ipfs (client) to work with an IPFS node hosted on ipfs.fluence.dev # instruct aqua-ipfs (client) to work with an IPFS node hosted on ipfs.fluence.dev
@ -75,9 +69,10 @@ ENV FLUENCE_ENV_CONNECTOR_CONTRACT_ADDRESS=0xb497e025D3095A197E30Ca84DEc36a637E6
# find deals from this block # find deals from this block
ENV FLUENCE_ENV_CONNECTOR_FROM_BLOCK=0x75f3fbc ENV FLUENCE_ENV_CONNECTOR_FROM_BLOCK=0x75f3fbc
# download fluence, builtins # download rust-peer binary, builtins
RUN --mount=type=bind,source=fluence,target=/fluence /fluence/download_builtins.sh /fluence/services.json COPY fluence/ /fluence/
RUN --mount=type=bind,source=fluence,target=/fluence /fluence/download_fluence.sh /fluence/fluence.json RUN /fluence/download_builtins.sh /fluence/services.json
RUN /fluence/download_fluence.sh /fluence/fluence.json
# copy default fluence config # copy default fluence config
COPY fluence/Config.default.toml /.fluence/v1/Config.toml COPY fluence/Config.default.toml /.fluence/v1/Config.toml
@ -92,6 +87,8 @@ COPY s6/minimal/ /
# ipfs # ipfs
# ---------------------------------------------------------------------------- # ----------------------------------------------------------------------------
FROM minimal as ipfs FROM minimal as ipfs
ARG TARGETPLATFORM
ARG BUILDPLATFORM
LABEL org.opencontainers.image.description="rust-peer bundled with IPFS daemon" LABEL org.opencontainers.image.description="rust-peer bundled with IPFS daemon"
LABEL dev.fluence.bundles.ipfs="${IPFS_VERSION}" LABEL dev.fluence.bundles.ipfs="${IPFS_VERSION}"
@ -110,8 +107,9 @@ ENV IPFS_ADDRESSES_ANNOUNCE=/ip4/127.0.0.1/tcp/4001,/ip4/127.0.0.1/tcp/4001/ws
ENV FLUENCE_ENV_AQUA_IPFS_EXTERNAL_API_MULTIADDR=/ip4/127.0.0.1/tcp/5001 ENV FLUENCE_ENV_AQUA_IPFS_EXTERNAL_API_MULTIADDR=/ip4/127.0.0.1/tcp/5001
ENV FLUENCE_ENV_AQUA_IPFS_LOCAL_API_MULTIADDR=/ip4/127.0.0.1/tcp/5001 ENV FLUENCE_ENV_AQUA_IPFS_LOCAL_API_MULTIADDR=/ip4/127.0.0.1/tcp/5001
# download fs-repo-migrations # download ipfs fs-repo-migrations tool
RUN wget -qO - "https://dist.ipfs.io/fs-repo-migrations/v2.0.2/fs-repo-migrations_v2.0.2_linux-amd64.tar.gz" | tar -C /usr/local/bin --strip-components=1 -zxvf - COPY docker/download_ipfs_fs_repo_migrations.sh /docker/download_ipfs_fs_repo_migrations.sh
RUN /docker/download_ipfs_fs_repo_migrations.sh
# copy s6 configs # copy s6 configs
COPY s6/ipfs/ / COPY s6/ipfs/ /
@ -126,6 +124,8 @@ ARG CERAMIC_VERSION
ARG GLAZED_VERSION ARG GLAZED_VERSION
ARG GETH_VERSION ARG GETH_VERSION
ARG BITCOIN_CLI_VERSION ARG BITCOIN_CLI_VERSION
ARG TARGETPLATFORM
ARG BUILDPLATFORM
LABEL org.opencontainers.image.description="rust-peer bundled with IPFS, Ceramic CLI and other tools" LABEL org.opencontainers.image.description="rust-peer bundled with IPFS, Ceramic CLI and other tools"
LABEL dev.fluence.image.bundles.ceramic="${CERAMIC_VERSION}" LABEL dev.fluence.image.bundles.ceramic="${CERAMIC_VERSION}"
@ -137,24 +137,17 @@ LABEL dev.fluence.image.bundles.geth="${GETH_VERSION}"
RUN curl -fsSL https://deb.nodesource.com/gpgkey/nodesource.gpg.key | gpg --dearmor > /usr/share/keyrings/nodesource.gpg \ RUN curl -fsSL https://deb.nodesource.com/gpgkey/nodesource.gpg.key | gpg --dearmor > /usr/share/keyrings/nodesource.gpg \
&& echo "deb [signed-by=/usr/share/keyrings/nodesource.gpg] https://deb.nodesource.com/node_16.x focal main" > /etc/apt/sources.list.d/nodesource.list && echo "deb [signed-by=/usr/share/keyrings/nodesource.gpg] https://deb.nodesource.com/node_16.x focal main" > /etc/apt/sources.list.d/nodesource.list
RUN \ RUN --mount=type=cache,target=/var/cache/apt \
echo "**** install packages ****" && \
apt-get update && \ apt-get update && \
apt-get install -y --no-install-recommends \ apt-get install -y --no-install-recommends \
musl \ musl \
nodejs && \ nodejs
echo "**** cleanup ****" && \
apt-get clean && \
rm -rf \
/tmp/* \
/var/lib/apt/lists/* \
/var/tmp/*
# install ceramic and glaze # install ceramic and glaze
RUN npm install --cache /cache --global \ RUN --mount=type=cache,target=/var/cache/npm \
npm install --cache /var/cache/npm --global \
@ceramicnetwork/cli@$CERAMIC_VERSION \ @ceramicnetwork/cli@$CERAMIC_VERSION \
@glazed/cli@$GLAZED_VERSION \ @glazed/cli@$GLAZED_VERSION
&& rm -rf /cache
# copy geth # copy geth
COPY --from=prepare-geth /usr/local/bin/geth /usr/bin/geth COPY --from=prepare-geth /usr/local/bin/geth /usr/bin/geth

15
docker/download_bitcoin_cli.sh Executable file
View File

@ -0,0 +1,15 @@
#! /usr/bin/env sh
case "$TARGETPLATFORM" in
'linux/amd64')
ARCHIVE="bitcoin-${BITCOIN_CLI_VERSION}-x86_64-linux-gnu.tar.gz"
;;
'linux/arm64')
ARCHIVE="bitcoin-${BITCOIN_CLI_VERSION}-aarch64-linux-gnu.tar.gz"
;;
esac
wget "https://bitcoincore.org/bin/bitcoin-core-${BITCOIN_CLI_VERSION}/$ARCHIVE"
grep " $ARCHIVE\$" SHA256SUMS | sha256sum -c -
tar -xzf "$ARCHIVE"
rm "$ARCHIVE"

View File

@ -0,0 +1,12 @@
#! /usr/bin/env sh
case "$TARGETPLATFORM" in
'linux/amd64')
ARCHIVE="fs-repo-migrations_v2.0.2_linux-amd64.tar.gz"
;;
'linux/arm64')
ARCHIVE="fs-repo-migrations_v2.0.2_linux-arm64.tar.gz"
;;
esac
wget -qO - "https://dist.ipfs.io/fs-repo-migrations/v2.0.2/$ARCHIVE" | tar -C /usr/local/bin --strip-components=1 -zxvf -

14
docker/install_libssl.sh Executable file
View File

@ -0,0 +1,14 @@
#! /usr/bin/env sh
case "$TARGETPLATFORM" in
'linux/amd64')
URL="http://security.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2_amd64.deb"
;;
'linux/arm64')
URL="http://launchpadlibrarian.net/668086110/libssl1.1_1.1.1-1ubuntu2.1~18.04.23_arm64.deb"
;;
esac
wget $URL -O libssl.deb
dpkg -i libssl.deb
rm libssl.deb

View File

@ -1,24 +1,62 @@
#!/usr/bin/env bash #!/usr/bin/env bash
set -o pipefail -o errexit -o nounset set -o pipefail -o errexit -o nounset
# Validate necessary commands and given config file
for cmd in jq curl sha256sum; do
if ! command -v $cmd &> /dev/null; then
echo "$cmd could not be found, please install it."
exit
fi
done
CONFIG="${1:-fluence.json}" CONFIG="${1:-fluence.json}"
jq -r '.url, .sha256, .version' $CONFIG | if [[ ! -f "$CONFIG" ]]; then
while echo "Config file $CONFIG not found!"
IFS=''
read -r url
read -r sha256
read -r version
do
echo "*** download $version ***"
# TODO: use --fail-with-body
curl -sL --fail $url -o /usr/bin/fluence || (
echo "failed to download $url" >&2
exit 1 exit 1
) fi
echo "$sha256 /usr/bin/fluence" | sha256sum --check --status || (
echo "incorrect SHA256" >&2 # Detect system architecture
ARCHITECTURE=$(uname -m)
# Map common architecture names to the ones used in your JSON
case "$ARCHITECTURE" in
x86_64)
KEY="x86_64"
;;
aarch64)
KEY="aarch64"
;;
*)
echo "Unsupported architecture: $ARCHITECTURE. Exiting..."
exit 1 exit 1
) ;;
chmod +x /usr/bin/fluence esac
done
# Fetch URL, SHA256, and version based on detected architecture
URL=$(jq -r ".${KEY}.url" $CONFIG)
SHA256=$(jq -r ".${KEY}.sha256" $CONFIG)
VERSION=$(jq -r ".${KEY}.version" $CONFIG)
echo "*** Downloading rust-peer version $VERSION for $ARCHITECTURE ***"
ATTEMPTS=5
while ((ATTEMPTS)); do
curl -sL --fail $URL -o /usr/bin/rust-peer && break
((ATTEMPTS--))
sleep 5
done
if ! ((ATTEMPTS)); then
echo "Failed to download $URL after 5 attempts. Exiting..."
exit 1
fi
if ! echo "$SHA256 /usr/bin/rust-peer" | sha256sum --check --status; then
echo "Incorrect SHA256 for the downloaded file. Exiting..."
exit 1
fi
chmod +x /usr/bin/rust-peer
echo "*** Successfully installed rust-peer version $VERSION for $ARCHITECTURE ***"

View File

@ -1,5 +1,12 @@
{ {
"x86_64": {
"version": "0.12.1", "version": "0.12.1",
"url": "https://github.com/fluencelabs/rust-peer/releases/download/rust-peer-v0.12.1/rust-peer", "url": "https://github.com/fluencelabs/rust-peer/releases/download/rust-peer-v0.12.1/rust-peer",
"sha256": "b17037511a020c28657f274f383ab6ead05da0aa5bc3eabbd60267ddf50a145d" "sha256": "b17037511a020c28657f274f383ab6ead05da0aa5bc3eabbd60267ddf50a145d"
},
"aarch64": {
"version": "0.12.1",
"url": "https://github.com/fluencelabs/rust-peer/releases/download/rust-peer-v0.12.1/rust-peer",
"sha256": "b17037511a020c28657f274f383ab6ead05da0aa5bc3eabbd60267ddf50a145d"
}
} }

View File

@ -5,7 +5,7 @@
with-contenv with-contenv
# remove old HOME variable from the environment, so the daemon isn't confused by it # remove old HOME variable from the environment, so the daemon isn't confused by it
# see https://github.com/fluencelabs/node-distro/issues/14 for more details # see https://github.com/fluencelabs/nox-distro/issues/14 for more details
unexport HOME unexport HOME
# 'setuidgid abc' runs '/usr/bin/fluence' as user 'abc' # 'setuidgid abc' runs '/usr/bin/nox' as user 'abc'
s6-setuidgid abc /usr/bin/fluence $@ s6-setuidgid abc /usr/bin/nox $@