Merge pull request #2 from hadim/docker

Build installer on Docker

.gitignore

@@ -1 +1 @@
-Miniforge3-*
+build/

.travis.yml

@@ -1,11 +1,21 @@
-language: minimal
+language: generic
 
-install:
-  - ./build.sh
+matrix:
+  include:
+    - os: linux
+      env:
+        - ARCH=aarch64
+        - DOCKERIMAGE=condaforge/linux-anvil-aarch64
+        - QEMU_BINARY=qemu-aarch64-static
+
+script:
+  - bash build_miniforge.sh
 
 deploy:
   provider: releases
-  file: Miniconda3-*.sh
+  file:
+    - "build/Miniforge*.sh"
+    - "build/Miniforge*.sha256"
   skip_cleanup: true
   api_key:
     secure: APK2p2eTV4YxOrQzwm+GnJs32V1MplI4VG8Lslr/y7qky8Cpl91o7pUAYOEs5dL2RZlPrDNiWNsPP5JYRiO7YjTv5lPXjSyoAavf/JcLIC1OgjNqn029wdITrOLXAPGtL3OM+K70CYdzbLc/X0VlC0mWSOUlxox7hoKKUhXY/EF29lneRjNhUM+Wb4utn61K70YzXvSGMnmHyti0KhOG7ACfZlJY99wwgQVBaAfKCCgFj+J1aeorifu6FgOu13cPfvnfnC9qJnd7mL9tTQl9ZjLTG8k8fvlyyqkEoExhSXpPzgp1a7q11uxT/ys8rR/2ivOMpYyGUgA848Eswiy9+w581An2TRMbm0zYdqzFG0kYTCcG22TGdZukKq1rXO6TDLCYT3V44L7s72VxfI4tNDCVDcfeQVOEUwlUmyC/O9CslKyZewf9gB5OT5IQ8eSDsrwoiGvQC7r2oxsm7zL4oOkC4mDpdRqOFeQOw3AkoRcrTSkw+eQALfhGEtGkJAOqEEHlsStSCBBkfokmRRksD+l6mSJKR426kDKDr3/69Taq5WiEzbTSGZyRk+5l7+GlNnolKXMrceA9sUHlMLSKvePmiPRdqq1jJLZHtlxEGRZyc7QolgRMpQGdQYxucearmM0BBPGahVsx+V4Mq5Jufom//P0isH9MJzk4p9npSIk=

Miniforge3/construct.yaml

@@ -1,16 +1,16 @@
 name: Miniforge3
+version: 4.7.11-0
 
 channels:
   - https://conda.anaconda.org/conda-forge
-  - https://conda.anaconda.org/anaconda
   # This will continue to be a dependency until the compilers are
-  # moved to conda-forge or to anaconda
+  # moved to conda-forge.
   - https://conda.anaconda.org/c4aarch64   [linux-aarch64]
 
 write_condarc: True
 license_file: ../LICENSE
 
 specs:
-  - python >=3,<3.8
-  - conda
+  - python 3.7
+  - conda 4.7.11
   - pip

README.md

@@ -1,21 +1,69 @@
-This repository holds a minimal installer for conda on platforms that
-conda-forge supports but that aren't supported by Miniconda
+# miniforge
+[![Build Status](https://travis-ci.org/conda-forge/miniforge.svg?branch=master)](https://travis-ci.org/conda-forge/miniforge)
 
+This repository holds a minimal installer for conda on platforms that conda-forge supports but that aren't supported by [Miniconda]((https://docs.conda.io/en/latest/miniconda.html).
 
-This work is still very experimental
+**Important: This work is still very experimental.**
 
-Relevant conversations
-* https://github.com/conda-forge/conda-forge.github.io/issues/871#issue-496677528
-* https://github.com/conda-forge/conda-forge.github.io/pull/922
+Relevant conversations:
 
+- https://github.com/conda-forge/conda-forge.github.io/issues/871#issue-496677528
+- https://github.com/conda-forge/conda-forge.github.io/pull/922
 
-Features of this repository to implement:
-- [ ] Automatic build of constructor
-    - [ ] aarch64 -- https://github.com/conda-forge/miniforge/pull/1
-    - [ ] Other architectures???
-- [ ] Automatic upload of constructor results
-    - [ ] Test: https://github.com/conda-forge/miniforge/pull/1
-- [ ] Automatic testing of constructor on Ubuntu 18.04 docker image
-- [ ] Integration with conda-forge's developer documentation
-- [ ] Integration with conda-forge's official site
-- [ ] Upstream to Anaconda???
+## Download
+
+Miniforge installers are available here: https://github.com/conda-forge/miniforge/releases.
+
+## Supported architectures
+
+- `aarch64` (also called `arm64`)
+
+## Features
+
+- [X] Automatic build of constructor.
+- [X] Automatic upload of constructor results.
+- [X] Automatic testing of constructor.
+- [ ] Integration with conda-forge's developer documentation.
+- [ ] Integration with conda-forge's official site.
+- [ ] Upstream to Anaconda ?
+
+## Testing
+
+After construction on Travis, the installer is tested against a range of distribution that match the installer architecture (`$ARCH`). For example when architecture is `aarch64`, the constructed installer is tested against:
+
+- Centos 7
+- Debian Buster (10)
+- Ubuntu 16.04
+- Ubuntu 18.04
+- Ubuntu 19.10
+
+## Usage
+
+Installers are built and uploaded via Travis but if you want to construct your own Miniforge installer, here is how:
+
+```bash
+# Configuration
+export ARCH=aarch64
+export DOCKERIMAGE=condaforge/linux-anvil-aarch64
+export QEMU_BINARY=qemu-aarch64-static
+
+bash build_miniforge.sh
+```
+
+## Release
+
+To release a new version of Miniforge:
+
+- Open a new PR.
+- Edit [`Miniforge3/construct.yaml`](./Miniforge3/construct.yaml).
+  - Update the `version` tag to: `$CONDA_VERSION-$BUILD_NUMBER`. For example, `4.7.11-0`.
+  - Update `conda` version in the `specs` section to `$CONDA_VERSION`.
+  - Update `python` version in the `specs` section to the Python version used in [Miniconda](https://repo.anaconda.com/miniconda/).
+- Once CI is happy, merge into `master`.
+- Tag `HEAD` to `$CONDA_VERSION-$BUILD_NUMBER` and push the tag:
+  - `git tag $CONDA_VERSION-$BUILD_NUMBER`
+  - `git push origin master --tags`
+
+## License
+
+[BDS 3-clause](./LICENSE)

build_miniforge.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+set -e
+
+# Check parameters
+ARCH=${ARCH:-aarch64}
+DOCKERIMAGE=${DOCKERIMAGE:-condaforge/linux-anvil-aarch64}
+QEMU_BINARY=${QEMU_BINARY:-qemu-aarch64-static}
+
+echo "============= Create build directory ============="
+mkdir -p build/
+chmod 777 build/
+
+echo "============= Enable QEMU ============="
+docker run --rm --privileged multiarch/qemu-user-static:register --reset --credential yes
+
+echo "============= Build the installer ============="
+docker run --rm -ti -v $(pwd):/construct $DOCKERIMAGE /construct/scripts/build.sh
+
+echo "============= Download QEMU static binaries ============="
+bash scripts/get_qemu.sh
+
+echo "============= Test the installer ============="
+for DOCKERFILE_PATH in $(find test_images/ -name "*.$ARCH")
+do
+  TEST_IMAGE_SUFFIX=$(echo $DOCKERFILE_PATH  | cut -d'.' -f2-)
+  TEST_IMAGE_NAME="miniforge_test_image.$TEST_IMAGE_SUFFIX"
+
+  echo "============= Building $TEST_IMAGE_NAME ============="
+  docker build -t $TEST_IMAGE_NAME -f $DOCKERFILE_PATH .
+
+  echo "============= Test installer on $TEST_IMAGE_NAME ============="
+  docker run --rm -ti -v $(pwd):/construct $TEST_IMAGE_NAME /construct/scripts/test.sh
+done

scripts/build.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+set -e
+
+echo "***** Start: Building Miniforge installer *****"
+
+CONSTRUCT_ROOT="${CONSTRUCT_ROOT:-/construct}"
+
+cd $CONSTRUCT_ROOT
+
+# Constructor should be >= 3.0.1 for aarch64.
+# See https://github.com/conda-forge/miniforge/pull/2#issuecomment-554394343
+echo "***** Install constructor *****"
+conda install -y "constructor>=3.0.1"
+conda list
+
+echo "***** Make temp directory *****"
+TEMP_DIR=$(mktemp -d)
+
+echo "***** Copy file for installer construction *****"
+cp -R Miniforge3/ $TEMP_DIR/
+cp LICENSE $TEMP_DIR
+
+echo "***** Construct the installer *****"
+constructor $TEMP_DIR/Miniforge3/ --output-dir $TEMP_DIR
+
+echo "***** Generate installer hash *****"
+cd $TEMP_DIR
+# This line ill break if there is more than one installer in the folder.
+INSTALLER_PATH=$(find . -name "Miniforge*.sh" | head -n 1)
+HASH_PATH="$INSTALLER_PATH.sha256"
+sha256sum $INSTALLER_PATH > $HASH_PATH
+
+echo "***** Move installer and hash to build folder *****"
+mv $INSTALLER_PATH $CONSTRUCT_ROOT/build/
+mv $HASH_PATH $CONSTRUCT_ROOT/build/
+
+echo "***** Done: Building Miniforge installer *****"

scripts/get_qemu.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+CURRENT_DIR=$(pwd)
+QEMU_STATIC_VERSION=v3.1.0-3
+QEMU_DIR="./build/qemu/"
+
+qemu_ppc64le_sha256=d018b96e20f7aefbc50e6ba93b6cabfd53490cdf1c88b02e7d66716fa09a7a17
+qemu_aarch64_sha256=a64b39b8ce16e2285cb130bcba7143e6ad2fe19935401f01c38325febe64104b
+qemu_arm_sha256=f4184c927f78d23d199056c5b0b6d75855e298410571d65582face3159117901
+
+mkdir -p $QEMU_DIR/
+cd $QEMU_DIR/
+
+wget https://github.com/multiarch/qemu-user-static/releases/download/${QEMU_STATIC_VERSION}/qemu-ppc64le-static
+wget https://github.com/multiarch/qemu-user-static/releases/download/${QEMU_STATIC_VERSION}/qemu-aarch64-static
+wget https://github.com/multiarch/qemu-user-static/releases/download/${QEMU_STATIC_VERSION}/qemu-arm-static
+
+sha256sum qemu-ppc64le-static | grep -F "${qemu_ppc64le_sha256}"
+sha256sum qemu-aarch64-static | grep -F "${qemu_aarch64_sha256}"
+sha256sum qemu-arm-static | grep -F "${qemu_aarch64_sha256}"
+
+chmod +x qemu-ppc64le-static
+chmod +x qemu-aarch64-static
+chmod +x qemu-arm-static
+
+cd $CURRENT_DIR

scripts/test.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+
+set -e
+
+echo "***** Start: Testing Miniforge installer *****"
+
+export CONDA_PATH="$HOME/miniforge"
+
+echo "***** Install dependencies *****"
+if [ -f /etc/redhat-release ]; then
+  yum install -y bzip2
+fi
+
+if [ -f /etc/debian_version ]; then
+  apt update -y
+  apt install -y bzip2
+fi
+
+cd /construct
+
+echo "***** Get the installer *****"
+INSTALLER_PATH=$(find build/ -name "Miniforge*$ARCH.sh" | head -n 1)
+
+echo "***** Run the installer *****"
+chmod +x $INSTALLER_PATH
+bash $INSTALLER_PATH -b -p $CONDA_PATH
+
+echo "***** Setup conda *****"
+source $CONDA_PATH/bin/activate
+
+echo "***** Print conda info *****"
+conda info
+
+echo "***** Run conda update *****"
+conda update --all -y
+
+echo "***** Python path *****"
+python -c "import sys; print(sys.executable)"
+python -c "import sys; assert 'miniforge' in sys.executable"
+
+echo "***** Print system informations from Python *****"
+python -c "print('Hello Miniforge !')"
+python -c "import platform; print(platform.architecture())"
+python -c "import platform; print(platform.system())"
+python -c "import platform; print(platform.machine())"
+python -c "import platform; print(platform.release())"
+
+echo "***** Done: Building Testing installer *****"

test_images/Docker.centos7.aarch64

@@ -0,0 +1,3 @@
+FROM arm64v8/centos:7
+ADD build/qemu/qemu-aarch64-static /usr/bin/qemu-aarch64-static
+ENV ARCH=aarch64

test_images/Docker.debian_buster.aarch64

@@ -0,0 +1,3 @@
+FROM arm64v8/debian:buster
+ADD build/qemu/qemu-aarch64-static /usr/bin/qemu-aarch64-static
+ENV ARCH=aarch64

test_images/Docker.ubuntu1604.aarch64

@@ -0,0 +1,3 @@
+FROM arm64v8/ubuntu:16.04
+ADD build/qemu/qemu-aarch64-static /usr/bin/qemu-aarch64-static
+ENV ARCH=aarch64

test_images/Docker.ubuntu1804.aarch64

@@ -0,0 +1,3 @@
+FROM arm64v8/ubuntu:18.04
+ADD build/qemu/qemu-aarch64-static /usr/bin/qemu-aarch64-static
+ENV ARCH=aarch64

test_images/Docker.ubuntu1910.aarch64

@@ -0,0 +1,3 @@
+FROM arm64v8/ubuntu:19.10
+ADD build/qemu/qemu-aarch64-static /usr/bin/qemu-aarch64-static
+ENV ARCH=aarch64