From 93f6a9b91b7ea3e100a9e5ef2d108d3950fa0202 Mon Sep 17 00:00:00 2001 From: YAMAMOTO Yuji Date: Wed, 11 Mar 2020 18:39:59 +0900 Subject: [PATCH] enable testing on Android x86_64 Currently, at the time of 2020/03/11, testing on AArch64 is not supported in the upstream repository --- .gitignore | 4 ++ Cargo.lock | 9 +-- ci/README.md | 3 + ci/android-install-ndk.sh | 19 ++++++ ci/android-install-sdk.sh | 73 ++++++++++++++++++++++ ci/android-sysimage.sh | 56 +++++++++++++++++ ci/docker/aarch64-linux-android/Dockerfile | 53 ++++++++++++++++ ci/docker/x86_64-linux-android/Dockerfile | 31 +++++++++ ci/run-docker.sh | 35 +++++++++++ ci/runtest-android.rs | 60 ++++++++++++++++++ 10 files changed, 339 insertions(+), 4 deletions(-) create mode 100644 ci/README.md create mode 100644 ci/android-install-ndk.sh create mode 100644 ci/android-install-sdk.sh create mode 100644 ci/android-sysimage.sh create mode 100644 ci/docker/aarch64-linux-android/Dockerfile create mode 100644 ci/docker/x86_64-linux-android/Dockerfile create mode 100755 ci/run-docker.sh create mode 100644 ci/runtest-android.rs diff --git a/.gitignore b/.gitignore index 1dfea1e5e..fc0b3b143 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,7 @@ install/ capi/ api-docs/ api-docs-repo/ + +# Generated by tests on Android +/avd +/core diff --git a/Cargo.lock b/Cargo.lock index c2dda2348..d2c14fc34 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -748,8 +748,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.67" -source = "git+https://github.com/rust-lang/libc.git#b22922f7d86a7e5cccc7bc039a375b23d11436b4" +version = "0.2.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dea0c0405123bba743ee3f91f49b1c7cfb684eef0da0a50110f758ccf24cdff0" [[package]] name = "llvm-sys" @@ -1775,9 +1776,9 @@ dependencies = [ [[package]] name = "wabt-sys" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af5d153dc96aad7dc13ab90835b892c69867948112d95299e522d370c4e13a08" +checksum = "23d7043ebb3e5d96fad7a8d3ca22ee9880748ff8c3e18092cfb2a49d3b8f9084" dependencies = [ "cc", "cmake", diff --git a/ci/README.md b/ci/README.md new file mode 100644 index 000000000..429bc7915 --- /dev/null +++ b/ci/README.md @@ -0,0 +1,3 @@ +# About this directory + +This directory is originally copied from [rust-lang/libc/ci](https://github.com/rust-lang/libc/tree/master/ci). diff --git a/ci/android-install-ndk.sh b/ci/android-install-ndk.sh new file mode 100644 index 000000000..59d3a1bbb --- /dev/null +++ b/ci/android-install-ndk.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env sh +# Copyright 2016 The Rust Project Developers. See the COPYRIGHT +# file at the top-level directory of this distribution and at +# http://rust-lang.org/COPYRIGHT. +# +# Licensed under the Apache License, Version 2.0 or the MIT license +# , at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + +set -ex + +NDK=android-ndk-r19c +curl --retry 20 -O https://dl.google.com/android/repository/${NDK}-linux-x86_64.zip +unzip -q -d ndk ${NDK}-linux-x86_64.zip +mv ./ndk/"$NDK"/* ./ndk/ + +rm -rf ./${NDK}-linux-x86_64.zip diff --git a/ci/android-install-sdk.sh b/ci/android-install-sdk.sh new file mode 100644 index 000000000..7f2104000 --- /dev/null +++ b/ci/android-install-sdk.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env sh +# Copyright 2016 The Rust Project Developers. See the COPYRIGHT +# file at the top-level directory of this distribution and at +# http://rust-lang.org/COPYRIGHT. +# +# Licensed under the Apache License, Version 2.0 or the MIT license +# , at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + +set -ex + +# Prep the SDK and emulator +# +# Note that the update process requires that we accept a bunch of licenses, and +# we can't just pipe `yes` into it for some reason, so we take the same strategy +# located in https://github.com/appunite/docker by just wrapping it in a script +# which apparently magically accepts the licenses. + +SDK=4333796 +mkdir sdk +curl --retry 20 https://dl.google.com/android/repository/sdk-tools-linux-${SDK}.zip -O +unzip -q -d sdk sdk-tools-linux-${SDK}.zip + +case "$1" in + arm | armv7) + api=24 + image="system-images;android-${api};google_apis;armeabi-v7a" + ;; + aarch64) + api=24 + image="system-images;android-${api};google_apis;arm64-v8a" + ;; + i686) + api=28 + image="system-images;android-${api};default;x86" + ;; + x86_64) + api=28 + image="system-images;android-${api};default;x86_64" + ;; + *) + echo "invalid arch: $1" + exit 1 + ;; +esac; + +# Try to fix warning about missing file. +# See https://askubuntu.com/a/1078784 +mkdir -p /root/.android/ +echo '### User Sources for Android SDK Manager' >> /root/.android/repositories.cfg +echo '#Fri Nov 03 10:11:27 CET 2017 count=0' >> /root/.android/repositories.cfg + +# Print all available packages +# yes | ./sdk/tools/bin/sdkmanager --list --verbose + +# --no_https avoids +# javax.net.ssl.SSLHandshakeException: sun.security.validator.ValidatorException: No trusted certificate found +# +# | grep -v = || true removes the progress bar output from the sdkmanager +# which produces an insane amount of output. +yes | ./sdk/tools/bin/sdkmanager --licenses --no_https | grep -v = || true +yes | ./sdk/tools/bin/sdkmanager --no_https \ + "emulator" \ + "platform-tools" \ + "platforms;android-${api}" \ + "${image}" | grep -v = || true + +echo "no" | + ./sdk/tools/bin/avdmanager create avd \ + --name "${1}" \ + --package "${image}" | grep -v = || true diff --git a/ci/android-sysimage.sh b/ci/android-sysimage.sh new file mode 100644 index 000000000..9eabd7c8d --- /dev/null +++ b/ci/android-sysimage.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash + +# Copyright 2017 The Rust Project Developers. See the COPYRIGHT +# file at the top-level directory of this distribution and at +# http://rust-lang.org/COPYRIGHT. +# +# Licensed under the Apache License, Version 2.0 or the MIT license +# , at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + +set -ex + +URL=https://dl.google.com/android/repository/sys-img/android + +main() { + local arch="${1}" + local name="${2}" + local dest=/system + local td + td="$(mktemp -d)" + + apt-get install --no-install-recommends e2tools + + pushd "${td}" + curl --retry 5 -O "${URL}/${name}" + unzip -q "${name}" + + local system + system="$(find . -name system.img)" + mkdir -p ${dest}/{bin,lib,lib64} + + # Extract android linker and libraries to /system + # This allows android executables to be run directly (or with qemu) + if [ "${arch}" = "x86_64" ] || [ "${arch}" = "arm64" ]; then + e2cp -p "${system}:/bin/linker64" "${dest}/bin/" + e2cp -p "${system}:/lib64/libdl.so" "${dest}/lib64/" + e2cp -p "${system}:/lib64/libc.so" "${dest}/lib64/" + e2cp -p "${system}:/lib64/libm.so" "${dest}/lib64/" + else + e2cp -p "${system}:/bin/linker" "${dest}/bin/" + e2cp -p "${system}:/lib/libdl.so" "${dest}/lib/" + e2cp -p "${system}:/lib/libc.so" "${dest}/lib/" + e2cp -p "${system}:/lib/libm.so" "${dest}/lib/" + fi + + # clean up + apt-get purge --auto-remove -y e2tools + + popd + + rm -rf "${td}" +} + +main "${@}" diff --git a/ci/docker/aarch64-linux-android/Dockerfile b/ci/docker/aarch64-linux-android/Dockerfile new file mode 100644 index 000000000..31a21112f --- /dev/null +++ b/ci/docker/aarch64-linux-android/Dockerfile @@ -0,0 +1,53 @@ +FROM ubuntu:19.04 + +RUN dpkg --add-architecture i386 && \ + apt-get update && \ + apt-get install -y --no-install-recommends \ + file \ + curl \ + ca-certificates \ + python \ + unzip \ + expect \ + openjdk-8-jre \ + libstdc++6:i386 \ + libpulse0 \ + gcc \ + libc6-dev \ + make \ + cmake # cmake is necessary to build wabt + +WORKDIR /android/ +COPY android* /android/ + +ENV ANDROID_ARCH=aarch64 +ENV PATH=$PATH:/android/sdk/tools:/android/sdk/platform-tools + +RUN sh /android/android-install-ndk.sh $ANDROID_ARCH +RUN sh /android/android-install-sdk.sh $ANDROID_ARCH +ENV ANDROID_NDK_HOME=/android/ndk + +RUN mv /root/.android /tmp +RUN chmod 777 -R /tmp/.android +RUN chmod 755 /android/sdk/tools/* /android/sdk/emulator/qemu/linux-x86_64/* + +ENV PATH=$PATH:/rust/bin:$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/linux-x86_64/bin \ + CARGO_TARGET_AARCH64_LINUX_ANDROID_AR=aarch64-linux-android-ar \ + CARGO_TARGET_AARCH64_LINUX_ANDROID_LINKER=aarch64-linux-android28-clang++ \ + CC_aarch64_linux_android=aarch64-linux-android28-clang \ + CXX_aarch64_linux_android=aarch64-linux-android28-clang++ \ + CARGO_TARGET_AARCH64_LINUX_ANDROID_RUNNER=/tmp/runtest \ + HOME=/tmp + +ADD runtest-android.rs /tmp/runtest.rs + +ENTRYPOINT [ \ + "bash", \ + "-c", \ + # set SHELL so android can detect a 64bits system, see + # http://stackoverflow.com/a/41789144 + "SHELL=/bin/dash /android/sdk/emulator/emulator @aarch64 -no-window & \ + rustc /tmp/runtest.rs -o /tmp/runtest && \ + exec \"$@\"", \ + "--" \ +] diff --git a/ci/docker/x86_64-linux-android/Dockerfile b/ci/docker/x86_64-linux-android/Dockerfile new file mode 100644 index 000000000..d8f89b13f --- /dev/null +++ b/ci/docker/x86_64-linux-android/Dockerfile @@ -0,0 +1,31 @@ +FROM ubuntu:19.04 + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + gcc \ + libc-dev \ + python \ + unzip \ + make \ + cmake # cmake is necessary to build wabt + +WORKDIR /android/ +ENV ANDROID_ARCH=x86_64 +COPY android-install-ndk.sh /android/ +RUN sh /android/android-install-ndk.sh $ANDROID_ARCH +ENV ANDROID_NDK_HOME=/android/ndk/ + +# We do not run x86_64-linux-android tests on an android emulator. +# See ci/android-sysimage.sh for informations about how tests are run. +COPY android-sysimage.sh /android/ +RUN bash /android/android-sysimage.sh x86_64 x86_64-24_r07.zip + +ENV PATH=$PATH:/rust/bin:$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/linux-x86_64/bin \ + CARGO_TARGET_X86_64_LINUX_ANDROID_AR=x86_64-linux-android-ar \ + CARGO_TARGET_X86_64_LINUX_ANDROID_LINKER=x86_64-linux-android28-clang++ \ + CC_x86_64_linux_android=x86_64-linux-android28-clang \ + CXX_x86_64_linux_android=x86_64-linux-android28-clang++ \ + LD_LIBRARY_PATH=$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib/x86_64-linux-android/ \ + HOME=/tmp diff --git a/ci/run-docker.sh b/ci/run-docker.sh new file mode 100755 index 000000000..7746e370f --- /dev/null +++ b/ci/run-docker.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env sh + +# Small script to run tests for a target (or all targets) inside all the +# respective docker images. + +set -e + +echo "${HOME}" +pwd + +TARGET="${1}" +shift + +echo "Building docker container for target $target" + +# use -f so we can use ci/ as build context +image_tag=test-"$TARGET" +docker build -t "$image_tag" -f "ci/docker/${TARGET}/Dockerfile" ci/ +mkdir -p target + +set -x + +docker run \ + --rm \ + --user "$(id -u)":"$(id -g)" \ + --env CARGO_HOME=/cargo \ + --env CARGO_TARGET_DIR=/checkout/target \ + --volume "$(dirname "$(dirname "$(command -v cargo)")")":/cargo \ + --volume "$(rustc --print sysroot)":/rust:ro \ + --volume "$(pwd)":/checkout:ro \ + --volume "$(pwd)"/target:/checkout/target \ + --init \ + --workdir /checkout \ + "$image_tag" \ + sh -c "HOME=/tmp PATH=\$PATH:/rust/bin exec cargo test --target ${TARGET} $@" diff --git a/ci/runtest-android.rs b/ci/runtest-android.rs new file mode 100644 index 000000000..490440e7c --- /dev/null +++ b/ci/runtest-android.rs @@ -0,0 +1,60 @@ +use std::env; +use std::path::{Path, PathBuf}; +use std::process::Command; + +fn main() { + let args = env::args_os() + .skip(1) + .filter(|arg| arg != "--quiet") + .collect::>(); + assert_eq!(args.len(), 1); + let test = PathBuf::from(&args[0]); + + // required to run an executable depending on wabt-rs + let android_ndk_home = env::var("ANDROID_NDK_HOME").expect("Can't get ANDROID_NDK_HOME!"); + let path = format!("{}/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib/x86_64-linux-android/libc++_shared.so", android_ndk_home); + let libcpp_shared = Path::new(&path); + + let dst = Path::new("/data/local/tmp"); + let dst_exec = Path::new("/data/local/tmp").join(test.file_name().unwrap()); + + let status = Command::new("adb") + .arg("wait-for-device") + .status() + .expect("failed to run: adb wait-for-device"); + assert!(status.success()); + + let status = Command::new("adb") + .arg("push") + .arg(&test) + .arg(&libcpp_shared) + .arg(&dst) + .status() + .expect("failed to run: adb pushr"); + assert!(status.success()); + + let output = Command::new("adb") + .arg("shell") + .arg("LD_LIBRARY_PATH=/data/local/tmp/") + .arg(&dst_exec) + .output() + .expect("failed to run: adb shell"); + assert!(status.success()); + + println!( + "status: {}\nstdout ---\n{}\nstderr ---\n{}", + output.status, + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + + let stdout = String::from_utf8_lossy(&output.stdout); + stdout + .lines() + .find(|l| { + (l.starts_with("PASSED ") && l.contains(" tests")) || l.starts_with("test result: ok") + }) + .unwrap_or_else(|| { + panic!("failed to find successful test run"); + }); +}