Add build scripts & CI configuration

.gitlab-ci.yml

@@ -1,20 +1,106 @@
-image: ubuntu:16.04
+image: ubuntu:17.04
 
 stages:
-  - build_qemu
-  - release
+  - build
+  - assemble
+  - test
+  - deploy
 
 before_script:
    - apt-get update -qq
+   - apt-get install -y -qq $(grep -vE "^\s*#" build_tools/outside/dependencies.txt | tr "\n" " ") > /dev/null
+
+variables:
+  GIT_STRATEGY: clone
+  GIT_CHECKOUT: "true"
+
+
+
+
+# build the ROOT.fs and the BOOT.fs. Aditionally build qemu for testing.
+build_bootfs:
+  stage: build
+  script:
+    - build_tools/outside/build_bootfs.sh
+    - tar -cpf build/BOOT.fs.tar build/BOOT.fs
+    - tar -cpf build/ROOT.fs_kernel_modules.tar build/ROOT.fs_kernel_modules
+  artifacts:
+    paths:
+      - build/BOOT.fs.tar
+      - build/ROOT.fs_kernel_modules.tar
+
+build_rootfs:
+  stage: build
+  script:
+    - git checkout -B "$CI_COMMIT_REF_NAME" "$CI_COMMIT_SHA"
+    - git branch --set-upstream-to=origin/$CI_COMMIT_REF_NAME $CI_COMMIT_REF_NAME
+    - build_tools/outside/build_rootfs.sh
+    - tar -cpf build/ROOT.fs.tar build/ROOT.fs
+  artifacts:
+    paths:
+      - build/ROOT.fs.tar
 
 build_qemu:
-  stage: build_qemu
+  stage: build
+  script:
+      - build_tools/outside/build_qemu.sh
+  artifacts:
+    paths:
+      - build/qemu-xlnx.git
+
+
+# build apertus tools inside qemu with the image
+assemble_image:
+  stage: assemble
+  dependencies:
+    - build_bootfs
+    - build_rootfs
+  script:
+    - tar --same-owner -xf build/BOOT.fs.tar
+    - tar --same-owner -xf build/ROOT.fs_kernel_modules.tar
+    - tar --same-owner -xf build/ROOT.fs.tar
+    - build_tools/outside/assemble_image.sh
+  artifacts:
+    paths:
+      - build/IMAGE.dd
+
+
+# Test the image using qemu
+test_qemu:
+  stage: test
+  dependencies:
+    - build_qemu
+    - build_bootfs
+    - assemble_image
+  script:
+    - tar --same-owner -xf build/BOOT.fs.tar
+    - tar --same-owner -xf build/ROOT.fs_kernel_modules.tar
+    - cd build/ && ../build_tools/outside/run_qemu.expect
+
+
+# create final artifacts & upload the link to gitlab pages
+pages:
+  stage: deploy
+  only:
+    - master
+  before_script:
+    - "echo no install"
+  script:
+    - mkdir public
+    - echo "<meta http-equiv="refresh" content=\"0; URL=$CI_PROJECT_URL/-/jobs/artifacts/master/download?job=final_artifacts\">" > public/index.html
+  artifacts:
+    paths:
+      - public
+
+final_artifacts:
+  stage: deploy
+  dependencies:
+    - assemble_image
+  before_script:
+    - "echo no install"
   script:
-    - apt-get install -y -qq wget git e2fsprogs mtools build-essential gcc-arm-linux-gnueabi libglib2.0-dev zlib1g-dev dh-autoreconf flex bison > /dev/null
-    - build_tools/axiom_beta_build_image_Ubuntu.sh
+    - cp build/IMAGE.dd axiom-beta-$CI_COMMIT_SHA.img
   artifacts:
     paths:
-      - quemu-xlnx.git/
-
-  cache:
-    untracked: true
+      - "*.img"
+    name: "axiom-beta-${CI_COMMIT_SHA}"

README.md

@@ -1,18 +1,13 @@
 # Axiom Beta Software
-[![pipeline status](https://gitlab.com/apertus/beta-software/badges/master/pipeline.svg)](https://gitlab.com/apertus/beta-software/commits/master)
+[![pipeline status](https://gitlab.com/apertus/beta-software/badges/master/pipeline.svg)](https://gitlab.com/apertus/beta-software/pipelines/)
 
 Firmware required to boot & operate the [Apertus Axiom Beta Camera](https://www.apertus.org/axiom-beta).
 
 Detailed instructions on how to use the Firmware can be found in the [wiki](https://wiki.apertus.org/index.php/AXIOM_Beta/AXIOM_Beta_Software)
 
 ## Building & hacking around
 A great way to start hacking on the Beta Frimware is building it.
-
-The easiest way to build the Axiom Beta Firmware is to download the `gitlab-ci-multi-runner` and do a containerized build. When you have the [gitlab-ci-multi-runner installed](https://docs.gitlab.com/runner/install/) simply run:
-```
-gitlab-ci-multi-runner exec docker build
-```
-
+Build instructions can be found in the [`build_tools/README.md`](build_tools/README.md) file
 
 ## Structure of this Repository
 The Repository is divided in the following Parts:

build_tools/README.md

@@ -0,0 +1,28 @@
+# build_tools
+This directory contains tools for building the main firmware image, that gets put on the sdcard.
+
+## Build It!
+### ... with docker
+The build scripts are intendet to be run inside a docker container with `ubuntu:17.04` and the dependencies in the `dependencies.txt` file installed. 
+
+First you have to get a shell in your docker container:
+```
+docker run -it ubuntu:17.04 /bin/bash
+```
+After this, install git and clone this repo inside the container:
+```
+apt get update && apt install -y git
+git clone https://github.com/apertus-open-source-cinema/beta-software
+cd beta software
+```
+Then install the missing dependencies and start the build process:
+```
+build_tools/full_build_ubuntu.sh
+```
+
+
+### ... without containerisation
+The preffered and tested build environment is Ubuntu 17.04.
+Other Ubuntu installations should work as well, as long as they are new enough that `mke2fs` has a `-d` option (Ubuntu 16.04 and below dont work).
+
+To run the build, follow instructions of the docker section without creating the container.

build_tools/full_build_ubuntu.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+echo "starting the full build..."
+set -e 
+set -o pipefail
+cd $(dirname $(realpath $0))/../
+
+echo -e "\ninstalling requirements:\n"
+apt-get update -qq
+apt-get install -y -qq $(grep -vE "^\s*#" build_tools/outside/dependencies.txt | tr "\n" " ") > /dev/null
+
+
+echo -e "\nbuilding the rootfs:\n"
+build_tools/outside/build_rootfs.sh
+
+
+echo -e "\nbuilding the bootfs:\n"
+build_tools/outside/build_bootfs.sh
+
+
+echo -e "\nassamblying the image:\n"
+build_tools/outside/assemble_image.sh
+
+
+echo "build finished :)"

build_tools/inside/dependencies.txt

@@ -0,0 +1,16 @@
+# This file contains a list of all the archlinuxarm packages, that are needed inside the image
+# install them via: pacman --noconfirm $(grep -vE "^\s*#" dependencies.txt | tr "\n" " ")
+
+# things needed for setup
+base-devel
+git
+figlet
+
+# dependencies of tools
+python
+devmem
+
+# convenience tools
+vim
+nano
+python-pip

build_tools/inside/install.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+set -e 
+set -o pipefail
+cd /opt/beta-software
+
+# network config
+mv /etc/resolv.conf /etc/resolv.conf.bak
+echo "nameserver 185.121.177.177" > /etc/resolv.conf
+
+# configure pacman & do initial database sync
+sed -i 's/#IgnorePkg   =/IgnorePkg = linux linux-*/' /etc/pacman.conf
+pacman --noconfirm -Syu
+
+# install dependencies
+pacman --noconfirm -Syu
+pacman --noconfirm -S $(grep -vE "^\s*#" build_tools/inside/dependencies.txt | tr "\n" " ")
+
+# do the normal axiom-update procedure
+build_tools/inside/update.sh
+
+# setup hostname & set before login message
+echo "axiom-beta" > /etc/hostname
+echo "Apertus Axiom Beta Booted!" > /etc/issue
+echo "Login as apertus with password axiom." >> /etc/issue
+
+# setup users
+echo 'echo -e "\033[31;5municorns dont log in as root\033[0m"' >> /root/.profile
+
+PASS=axiom
+USERNAME=apertus
+useradd -p $(openssl passwd -1 $PASS) -d /home/"$USERNAME" -m -g users -s /bin/bash "$USERNAME"
+echo "$USERNAME      ALL=(ALL) ALL" >> /etc/sudoers
+
+# configure ssh
+echo "PermitRootLogin no" >> /etc/ssh/sshd_config
+echo "X11Forwarding yes" >> /etc/ssh/sshd_config
+
+# undo the temporary hacks
+mv /etc/resolv.conf.bak /etc/resolv.conf

build_tools/inside/update.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+set -e 
+set -o pipefail
+cd /opt/beta-software
+
+# pull newest code from git
+git pull
+
+# do the real update
+build_tools/inside/update_inner.sh

build_tools/inside/update_inner.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+set -e 
+set -o pipefail
+cd /opt/beta-software
+
+
+# make a sysupdate
+pacman --noconfirm -Syu
+
+
+# build all the tools
+for dir in $(find software/cmv_tools/ -maxdepth 1 -type d | tail -n +2); do (cd $dir && make && make install); done
+for dir in $(find software/processing_tools/ -maxdepth 1 -type d | tail -n +2); do (cd $dir && make && make install); done
+
+for script in software/scripts/*.sh; do ln -sf $(pwd)/$script /usr/local/bin/axiom-$(basename $script .sh); done
+for script in software/scripts/*.py; do ln -sf $(pwd)/$script /usr/local/bin/axiom-$(basename $script .py); done
+
+ln -sf $(pwd)/build_tools/inside/update.sh /usr/local/bin/axiom-update
+
+# TODO: build and install the control daemon
+
+# TODO: build the misc tools from: https://github.com/apertus-open-source-cinema/misc-tools-utilities/tree/master/raw2dng
+
+
+# Add login headers to users
+figlet "Axiom-Beta" > /etc/motd
+echo "Software version $(git rev-parse --short HEAD). Last updated on $(date +"%d.%m.%y %H:%M UTC")" >> /etc/motd
+echo "To update run, \"axiom-update\"." >> /etc/motd
+
+
+# finish the update
+echo "axiom-update finished. Software version is now $(git rev-parse --short HEAD)."