Attempt to fix image file generation by using Travis CI stages with Amazon AWS S3

.travis.yml

@@ -1,13 +1,23 @@
-language: c++
+env:
-compiler: gcc
+  global:
-sudo: true
+  - PATH=$HOME/.local/bin:$PATH
-cache: false
+  - AWS_BUCKET="ofxpimapper"
-matrix:
+  - AWS_SHARED="~/shared"
+
+git:
+  depth: 1
+
+before_install:
+- pip install --user awscli
+- mkdir -p "${AWS_SHARED}"
+
+jobs:
   include:
-  - os: linux
+  - stage: addons
+    os: linux
     dist: trusty
     sudo: required
-    env: TARGET="rpi" SCRIPT="scripts/ci/rpi/pisetup.sh" IMAGE="opm-${TRAVIS_TAG}.img" RPI_ZIP="raspbian_lite-2017-07-05-of_v0.9.8.img.zip" RPI_URL="https://github.com/kr15h/openframeworks-rpi-image/releases/download/v1.0.2/${RPI_ZIP}"
+    env: IMAGE="opm-${TRAVIS_TAG}.img" RPI_ZIP="raspbian_lite-2017-07-05-of_v0.9.8.img.zip" RPI_URL="https://github.com/kr15h/openframeworks-rpi-image/releases/download/v1.0.2/${RPI_ZIP}"
     addons:
       apt:
         packages:
@@ -19,8 +29,31 @@ matrix:
         - dosfstools
         - zip
     script:
-    - sudo bash ./scripts/ci/rpi/mkimage.sh
+    - sudo bash ./scripts/ci/rpi/addons.sh
-    - zip ${IMAGE}.zip ${IMAGE}
+    - zip "${AWS_SHARED}/${IMAGE}.zip" "./${IMAGE}"
+    - aws s3 rm "s3://${AWS_BUCKET}" --recursive
+    - aws s3 sync "${AWS_SHARED}" "s3://${AWS_BUCKET}"
+  - stage: finalize
+    os: linux
+    dist: trusty
+    sudo: required
+    env: IMAGE="opm-${TRAVIS_TAG}.img"
+    addons:
+      apt:
+        packages:
+        - qemu
+        - qemu-user-static
+        - binfmt-support
+        - parted
+        - wget
+        - dosfstools
+        - zip
+    script:
+    - aws s3 sync "s3://${AWS_BUCKET}" "${AWS_SHARED}"
+    - aws s3 rm "s3://${AWS_BUCKET}" --recursive
+    - mv "${AWS_SHARED}/${IMAGE}.zip" "./${IMAGE}.zip"
+    - sudo bash ./scripts/ci/rpi/finalize.sh
+    - zip "${IMAGE}.zip" "./${IMAGE}"
     deploy:
       provider: releases
       api_key:
@@ -31,39 +64,40 @@ matrix:
       on:
         tags: true
 
-  - os: linux
+#matrix:
-    dist: trusty
+#  include:
-    sudo: required
+#  - os: linux
-    env: TARGET="linux64" OF_BRANCH="master"
+#    dist: trusty
-    addons:
+#    sudo: required
-      apt:
+#    env: TARGET="linux64" OF_BRANCH="master"
-        sources:
+#    addons:
-        - ubuntu-toolchain-r-test
+#      apt:
-        packages:
+#        sources:
-        - gcc-4.9
+#        - ubuntu-toolchain-r-test
-        - g++-4.9
+#        packages:
-        - gdb
+#        - gcc-4.9
-  - os: osx
+#        - g++-4.9
-    osx_image: xcode8
+#        - gdb
-    compiler: clang
+#  - os: osx
-    env: TARGET="osx" OF_BRANCH="master"
+#    osx_image: xcode8
-  exclude:
+#    compiler: clang
-  - compiler: gcc
+#    env: TARGET="osx" OF_BRANCH="master"
-install:
+#  exclude:
-- |
+#  - compiler: gcc
-  if ! [ "${TARGET}" == "rpi" ]; then
+#install:
-    cd ~
+#- |
-    git clone --depth=1 --branch=$OF_BRANCH https://github.com/openframeworks/openFrameworks
+#  if ! [ "${TARGET}" == "rpi" ]; then
-    cd ~/openFrameworks/addons
+#    cd ~
-    git clone --depth=1 https://github.com/jeffcrouse/ofxJSON.git
+#    git clone --depth=1 --branch=$OF_BRANCH https://github.com/openframeworks/openFrameworks
-    cd ~/openFrameworks
+#    cd ~/openFrameworks/addons
-    scripts/ci/addons/install.sh
+#    git clone --depth=1 https://github.com/jeffcrouse/ofxJSON.git
-  fi
+#    cd ~/openFrameworks
-script:
+#    scripts/ci/addons/install.sh
-- |
+#  fi
-  if ! [ "${TARGET}" == "rpi" ]; then
+#script:
-    cd ~/openFrameworks
+#- |
-    scripts/ci/addons/build.sh
+#  if ! [ "${TARGET}" == "rpi" ]; then
-  fi
+#    cd ~/openFrameworks
-git:
+#    scripts/ci/addons/build.sh
-  depth: 1
+#  fi
+#

scripts/ci/rpi/addons-script.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+echo "Compiling addons."
+
+cd /home/pi/openFrameworks/addons/ofxOMXPlayer/example-basic
+make -j
+
+cd /home/pi/openFrameworks/addons/ofxJSON/example_file_read_write
+make -j
+
+echo "Compiling addons done!"

scripts/ci/rpi/mkimage.sh → scripts/ci/rpi/addons.sh

@@ -14,9 +14,7 @@ fi
 # Image creation constants.
 # See .travis.yml for environment variables.
 MOUNT="mnt"
-
+SCRIPT="addons-script.sh"
-echo "Welcome to mkimage.sh script."
-pwd
 
 # Unmount drives and general cleanup on exit, the trap ensures this will always
 # run execpt in the most extream cases.
@@ -36,10 +34,7 @@ cleanup() {
 trap cleanup EXIT
 
 # Download raspbian arm only if we have not already done so
-[ ! -f "${RPI_ZIP}" ] && wget "${RPI_URL}"
+[ ! -f "${RPI_ZIP}" ] && wget --progress=bar:force "${RPI_URL}"
-
-# Clean the existing image files
-(ls *.img >> /dev/null 2>&1 && rm *.img) || echo "no .img files to remove"
 
 # Unzip Raspbian
 # -u  update files, create if necessary

scripts/ci/rpi/pisetup.sh → scripts/ci/rpi/finalize-script.sh

@@ -1,14 +1,12 @@
 #!/bin/bash
 
-echo "Hello! Me is pisetup.sh script."
+echo "Finalizing OPM RPi image."
-
-sudo apt-get -yq install usbmount dosfstools exfat-fuse exfat-utils
-
-pwd
-df -h
 
 cd /home/pi/openFrameworks/addons/ofxPiMapper/example_basic
-make -j$(nproc)
+make -j
+
+echo "Inastalling extra packages."
+sudo apt-get -yq install usbmount dosfstools exfat-fuse exfat-utils
 
 echo "Setting hostname."
 echo "opm" > /etc/hostname
@@ -20,4 +18,4 @@ echo "@reboot /home/pi/openFrameworks/addons/ofxPiMapper/example_basic/bin/examp
 crontab mycron
 rm mycron
 
-echo "RPi setup done!"
+echo "OPM RPi image setup done!"

scripts/ci/rpi/finalize.sh

@@ -0,0 +1,72 @@
+#!/bin/bash
+
+# Setup script error handling see https://disconnected.systems/blog/another-bash-strict-mode for details
+set -xuo pipefail
+trap 's=$?; echo "$0: Error on line "$LINENO": $BASH_COMMAND"; exit $s' ERR
+IFS=$'\n\t'
+
+# Ensure we are root
+if [[ $EUID -ne 0 ]]; then
+    echo "This script must be run as root" 1>&2
+    exit 1
+fi
+
+# Image creation constants.
+# See .travis.yml for environment variables.
+MOUNT="mnt"
+SCRIPT="finalize-script.sh"
+
+# Unmount drives and general cleanup on exit, the trap ensures this will always
+# run execpt in the most extream cases.
+cleanup() {
+    [[ -f "${MOUNT}/tmp/${SCRIPT}" ]] && rm "${MOUNT}/tmp/${SCRIPT}"
+    if [[ -d "${MOUNT}" ]]; then
+        umount "${MOUNT}/dev/pts" || true
+        umount "${MOUNT}/dev" || true
+        umount "${MOUNT}/proc" || true
+        umount "${MOUNT}/sys" || true
+        umount "${MOUNT}/boot" || true
+        umount "${MOUNT}" || true
+        rmdir "${MOUNT}" || true
+    fi
+    [ -n "${loopdev:-}" ] && losetup --detach "${loopdev}" || true
+}
+trap cleanup EXIT
+
+# Unzip Raspbian
+# -u  update files, create if necessary
+unzip -u "${IMAGE}.zip"
+
+# Configure loopback device.
+loopdev=$(losetup --find --show "${IMAGE}")
+echo "Created loopback device ${loopdev}"
+
+echo "Mounting filesystem."
+bootdev="${loopdev}p1"
+rootdev="${loopdev}p2"
+partprobe "${loopdev}"
+
+[ ! -d "${MOUNT}" ] && mkdir "${MOUNT}"
+mount "${rootdev}" "${MOUNT}"
+[ ! -d "${MOUNT}/boot" ] && mkdir "${MOUNT}/boot"
+mount "${bootdev}" "${MOUNT}/boot"
+
+# Copy our installation script and other artifacts.
+install -Dm755 "${SCRIPT}" "${MOUNT}/tmp/${SCRIPT}"
+
+# Prep the chroot.
+mount --bind /proc "${MOUNT}/proc"
+mount --bind /sys "${MOUNT}/sys"
+mount --bind /dev "${MOUNT}/dev"
+mount --bind /dev/pts "${MOUNT}/dev/pts"
+
+cp /etc/resolv.conf "${MOUNT}/etc/resolv.conf"
+cp /usr/bin/qemu-arm-static "${MOUNT}/usr/bin"
+cp "${MOUNT}/etc/ld.so.preload" "${MOUNT}/etc/_ld.so.preload"
+echo "" > "${MOUNT}/etc/ld.so.preload"
+
+# Run the installation script as if we would be inside the Raspberry Pi.
+chroot "${MOUNT}" "/tmp/${SCRIPT}"
+
+# Put back the old ld.so.preload script.
+mv "${MOUNT}/etc/_ld.so.preload" "${MOUNT}/etc/ld.so.preload"