Attempt to fix image file generation by using Travis CI stages with Amazon AWS S3

7 years ago · a0dce821cd
5 changed files with 170 additions and 60 deletions
--- a/.travis.yml
+++ b/.travis.yml
@ -1,13 +1,23 @@
-language: c++
-compiler: gcc
-sudo: true
-cache: false
-matrix:
+env:
+  global:
+  - PATH=$HOME/.local/bin:$PATH
+  - AWS_BUCKET="ofxpimapper"
+  - AWS_SHARED="~/shared"
+
+git:
+  depth: 1
+
+before_install:
+- pip install --user awscli
+- mkdir -p "${AWS_SHARED}"
+
+jobs:
  include:
-  - os: linux
+  - stage: addons
+    os: linux
    dist: trusty
    sudo: required
-    env: TARGET="rpi" SCRIPT="scripts/ci/rpi/pisetup.sh" IMAGE="opm-${TRAVIS_TAG}.img" RPI_ZIP="raspbian_lite-2017-07-05-of_v0.9.8.img.zip" RPI_URL="https://github.com/kr15h/openframeworks-rpi-image/releases/download/v1.0.2/${RPI_ZIP}"
+    env: IMAGE="opm-${TRAVIS_TAG}.img" RPI_ZIP="raspbian_lite-2017-07-05-of_v0.9.8.img.zip" RPI_URL="https://github.com/kr15h/openframeworks-rpi-image/releases/download/v1.0.2/${RPI_ZIP}"
    addons:
      apt:
        packages:
@ -19,8 +29,31 @@ matrix:
        - dosfstools
        - zip
    script:
-    - sudo bash ./scripts/ci/rpi/mkimage.sh
-    - zip ${IMAGE}.zip ${IMAGE}
+    - sudo bash ./scripts/ci/rpi/addons.sh
+    - zip "${AWS_SHARED}/${IMAGE}.zip" "./${IMAGE}"
+    - aws s3 rm "s3://${AWS_BUCKET}" --recursive
+    - aws s3 sync "${AWS_SHARED}" "s3://${AWS_BUCKET}"
+  - stage: finalize
+    os: linux
+    dist: trusty
+    sudo: required
+    env: IMAGE="opm-${TRAVIS_TAG}.img"
+    addons:
+      apt:
+        packages:
+        - qemu
+        - qemu-user-static
+        - binfmt-support
+        - parted
+        - wget
+        - dosfstools
+        - zip
+    script:
+    - aws s3 sync "s3://${AWS_BUCKET}" "${AWS_SHARED}"
+    - aws s3 rm "s3://${AWS_BUCKET}" --recursive
+    - mv "${AWS_SHARED}/${IMAGE}.zip" "./${IMAGE}.zip"
+    - sudo bash ./scripts/ci/rpi/finalize.sh
+    - zip "${IMAGE}.zip" "./${IMAGE}"
    deploy:
      provider: releases
      api_key:
@ -31,39 +64,40 @@ matrix:
      on:
        tags: true

-  - os: linux
-    dist: trusty
-    sudo: required
-    env: TARGET="linux64" OF_BRANCH="master"
-    addons:
-      apt:
-        sources:
-        - ubuntu-toolchain-r-test
-        packages:
-        - gcc-4.9
-        - g++-4.9
-        - gdb
-  - os: osx
-    osx_image: xcode8
-    compiler: clang
-    env: TARGET="osx" OF_BRANCH="master"
-  exclude:
-  - compiler: gcc
-install:
- |
-  if ! [ "${TARGET}" == "rpi" ]; then
-    cd ~
-    git clone --depth=1 --branch=$OF_BRANCH https://github.com/openframeworks/openFrameworks
-    cd ~/openFrameworks/addons
-    git clone --depth=1 https://github.com/jeffcrouse/ofxJSON.git
-    cd ~/openFrameworks
-    scripts/ci/addons/install.sh
-  fi
-script:
- |
-  if ! [ "${TARGET}" == "rpi" ]; then
-    cd ~/openFrameworks
-    scripts/ci/addons/build.sh
-  fi
-git:
-  depth: 1
+#matrix:
+#  include:
+#  - os: linux
+#    dist: trusty
+#    sudo: required
+#    env: TARGET="linux64" OF_BRANCH="master"
+#    addons:
+#      apt:
+#        sources:
+#        - ubuntu-toolchain-r-test
+#        packages:
+#        - gcc-4.9
+#        - g++-4.9
+#        - gdb
+#  - os: osx
+#    osx_image: xcode8
+#    compiler: clang
+#    env: TARGET="osx" OF_BRANCH="master"
+#  exclude:
+#  - compiler: gcc
+#install:
+#- |
+#  if ! [ "${TARGET}" == "rpi" ]; then
+#    cd ~
+#    git clone --depth=1 --branch=$OF_BRANCH https://github.com/openframeworks/openFrameworks
+#    cd ~/openFrameworks/addons
+#    git clone --depth=1 https://github.com/jeffcrouse/ofxJSON.git
+#    cd ~/openFrameworks
+#    scripts/ci/addons/install.sh
+#  fi
+#script:
+#- |
+#  if ! [ "${TARGET}" == "rpi" ]; then
+#    cd ~/openFrameworks
+#    scripts/ci/addons/build.sh
+#  fi
+#
--- a/scripts/ci/rpi/addons-script.sh
+++ b/scripts/ci/rpi/addons-script.sh
@ -0,0 +1,11 @@
+#!/bin/bash
+
+echo "Compiling addons."
+
+cd /home/pi/openFrameworks/addons/ofxOMXPlayer/example-basic
+make -j
+
+cd /home/pi/openFrameworks/addons/ofxJSON/example_file_read_write
+make -j
+
+echo "Compiling addons done!"
--- a/scripts/ci/rpi/mkimage.sh
+++ b/scripts/ci/rpi/mkimage.sh
@ -14,9 +14,7 @@ fi
 # Image creation constants.
 # See .travis.yml for environment variables.
 MOUNT="mnt"
-
-echo "Welcome to mkimage.sh script."
-pwd
+SCRIPT="addons-script.sh"

 # Unmount drives and general cleanup on exit, the trap ensures this will always
 # run execpt in the most extream cases.
@ -36,10 +34,7 @@ cleanup() {
 trap cleanup EXIT

 # Download raspbian arm only if we have not already done so
-[ ! -f "${RPI_ZIP}" ] && wget "${RPI_URL}"
-
-# Clean the existing image files
-(ls *.img >> /dev/null 2>&1 && rm *.img) || echo "no .img files to remove"
+[ ! -f "${RPI_ZIP}" ] && wget --progress=bar:force "${RPI_URL}"

 # Unzip Raspbian
 # -u  update files, create if necessary
--- a/scripts/ci/rpi/finalize-script.sh
+++ b/scripts/ci/rpi/finalize-script.sh
@ -1,14 +1,12 @@
 #!/bin/bash

-echo "Hello! Me is pisetup.sh script."
-
-sudo apt-get -yq install usbmount dosfstools exfat-fuse exfat-utils
-
-pwd
-df -h
+echo "Finalizing OPM RPi image."

 cd /home/pi/openFrameworks/addons/ofxPiMapper/example_basic
-make -j$(nproc)
+make -j
+
+echo "Inastalling extra packages."
+sudo apt-get -yq install usbmount dosfstools exfat-fuse exfat-utils

 echo "Setting hostname."
 echo "opm" > /etc/hostname
@ -20,4 +18,4 @@ echo "@reboot /home/pi/openFrameworks/addons/ofxPiMapper/example_basic/bin/examp
 crontab mycron
 rm mycron

-echo "RPi setup done!"
+echo "OPM RPi image setup done!"
--- a/scripts/ci/rpi/finalize.sh
+++ b/scripts/ci/rpi/finalize.sh
@ -0,0 +1,72 @@
+#!/bin/bash
+
+# Setup script error handling see https://disconnected.systems/blog/another-bash-strict-mode for details
+set -xuo pipefail
+trap 's=$?; echo "$0: Error on line "$LINENO": $BASH_COMMAND"; exit $s' ERR
+IFS=$'\n\t'
+
+# Ensure we are root
+if [[ $EUID -ne 0 ]]; then
+    echo "This script must be run as root" 1>&2
+    exit 1
+fi
+
+# Image creation constants.
+# See .travis.yml for environment variables.
+MOUNT="mnt"
+SCRIPT="finalize-script.sh"
+
+# Unmount drives and general cleanup on exit, the trap ensures this will always
+# run execpt in the most extream cases.
+cleanup() {
+    [[ -f "${MOUNT}/tmp/${SCRIPT}" ]] && rm "${MOUNT}/tmp/${SCRIPT}"
+    if [[ -d "${MOUNT}" ]]; then
+        umount "${MOUNT}/dev/pts" || true
+        umount "${MOUNT}/dev" || true
+        umount "${MOUNT}/proc" || true
+        umount "${MOUNT}/sys" || true
+        umount "${MOUNT}/boot" || true
+        umount "${MOUNT}" || true
+        rmdir "${MOUNT}" || true
+    fi
+    [ -n "${loopdev:-}" ] && losetup --detach "${loopdev}" || true
+}
+trap cleanup EXIT
+
+# Unzip Raspbian
+# -u  update files, create if necessary
+unzip -u "${IMAGE}.zip"
+
+# Configure loopback device.
+loopdev=$(losetup --find --show "${IMAGE}")
+echo "Created loopback device ${loopdev}"
+
+echo "Mounting filesystem."
+bootdev="${loopdev}p1"
+rootdev="${loopdev}p2"
+partprobe "${loopdev}"
+
+[ ! -d "${MOUNT}" ] && mkdir "${MOUNT}"
+mount "${rootdev}" "${MOUNT}"
+[ ! -d "${MOUNT}/boot" ] && mkdir "${MOUNT}/boot"
+mount "${bootdev}" "${MOUNT}/boot"
+
+# Copy our installation script and other artifacts.
+install -Dm755 "${SCRIPT}" "${MOUNT}/tmp/${SCRIPT}"
+
+# Prep the chroot.
+mount --bind /proc "${MOUNT}/proc"
+mount --bind /sys "${MOUNT}/sys"
+mount --bind /dev "${MOUNT}/dev"
+mount --bind /dev/pts "${MOUNT}/dev/pts"
+
+cp /etc/resolv.conf "${MOUNT}/etc/resolv.conf"
+cp /usr/bin/qemu-arm-static "${MOUNT}/usr/bin"
+cp "${MOUNT}/etc/ld.so.preload" "${MOUNT}/etc/_ld.so.preload"
+echo "" > "${MOUNT}/etc/ld.so.preload"
+
+# Run the installation script as if we would be inside the Raspberry Pi.
+chroot "${MOUNT}" "/tmp/${SCRIPT}"
+
+# Put back the old ld.so.preload script.
+mv "${MOUNT}/etc/_ld.so.preload" "${MOUNT}/etc/ld.so.preload"