Merge pull request #24 from EESSI/build_packages

Build deb and rpm packages for clients

.github/workflows/build-client-packages.yml

@@ -0,0 +1,131 @@
+name: Build client packages
+
+on:
+  push:
+    branches:
+      - master
+      - build_packages
+    tags:
+      - 'v*'
+  pull_request:
+    branches:
+      - master
+      - build_packages
+
+jobs:
+
+  build-packages:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Get the version number for the packages
+        id: get_version
+        # If this is a tag, use the tag name (e.g. v1.2.3) without v as version number.
+        # Otherwise, just use 0.0.
+        run: |
+          VERSION=0.0
+          REF_NAME=${{ github.ref }}
+          [[ $REF_NAME == refs/tags/v* ]] && VERSION=${REF_NAME/refs\/tags\/v/}
+          echo ::set-output name=version::${VERSION}
+
+      # The next step uses a custom Ansible inventory, and due to that it cannot find
+      # the group_vars folder inside the inventory folder. This symlink fixes that.
+      - name: Make symlink to group_vars
+        run: ln -s inventory/group_vars
+
+      - name: Prepare package source
+        uses: roles-ansible/check-ansible-debian-stretch-action@master
+        with:
+          targets: "./prepare-client-packages.yml"
+          hosts: "localhost"
+
+      - name: Build RPM package
+        id: build-rpm
+        uses: bpicode/github-action-fpm@master
+        with:
+          fpm_args: "etc"
+          fpm_opts: "--debug -n cvmfs-config-esssi -v ${{ steps.get_version.outputs.version }} -t rpm -a all -s dir -C ./package --description 'CVMFS config repository package for EESSI.'"
+
+      - name: Build Deb package
+        id: build-deb
+        uses: bpicode/github-action-fpm@master
+        with:
+          fpm_args: "etc"
+          fpm_opts: "--debug -n cvmfs-config-esssi -v ${{ steps.get_version.outputs.version }} -t deb -a all -s dir -C ./package --description 'CVMFS config repository package for EESSI.'"
+
+      - name: Find filenames of downloaded packages
+        id: find_filenames
+        shell: bash
+        run: |
+          rpmfile="$(ls -1 *.rpm)"
+          debfile="$(ls -1 *.deb)"
+          echo ::set-output name=rpmfile::${rpmfile}
+          echo ::set-output name=debfile::${debfile}
+
+      - name: Upload Deb package as artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: Deb package
+          path: ${{ steps.find_filenames.outputs.debfile }}
+
+      - name: Upload RPM package as artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: RPM package
+          path: ${{ steps.find_filenames.outputs.rpmfile }}
+
+  release:
+    needs: build-packages
+    if: startsWith(github.ref, 'refs/tags/')
+    runs-on: ubuntu-latest
+    steps:
+      - name: Download Deb package
+        uses: actions/download-artifact@v2
+        with:
+          name: Deb package
+
+      - name: Download RPM package
+        uses: actions/download-artifact@v2
+        with:
+          name: RPM package
+
+      - name: Find filenames of downloaded packages
+        id: find_filenames
+        shell: bash
+        run: |
+          rpmfile="$(ls -1 *.rpm)"
+          debfile="$(ls -1 *.deb)"
+          echo ::set-output name=rpmfile::${rpmfile}
+          echo ::set-output name=debfile::${debfile}
+
+      - name: Create Release
+        id: create_release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          tag_name: ${{ github.ref }}
+          release_name: Filesystem Layer ${{ github.ref }}
+          draft: false
+          prerelease: false
+
+      - name: Upload RPM as release asset
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: ${{ steps.find_filenames.outputs.rpmfile }}
+          asset_name: ${{ steps.find_filenames.outputs.rpmfile }}
+          asset_content_type: application/x-rpm
+
+      - name: Upload Deb as release asset
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: ${{ steps.find_filenames.outputs.debfile }}
+          asset_name: ${{ steps.find_filenames.outputs.debfile }}
+          asset_content_type: application/x-debian-package

.gitignore

@@ -1,2 +1,3 @@
+build
 hosts
 roles/

README.md

@@ -135,6 +135,8 @@ ansible-playbook -b -K -e @inventory/local_site_specific_vars.yml localproxy.yml
 ```
 
 ### Clients
+
+#### Method 1: Ansible 
 Make sure that your hosts file contains the list of hosts where the CVMFS client should be installed.
 Furthermore, you can define a list of (local) proxy servers
 that your clients should use in `inventory/local_site_specific_vars.yml` using the parameter `local_cvmfs_http_proxies`.
@@ -146,6 +148,33 @@ Finally, run the playbook:
 ansible-playbook -b -K -e @inventory/local_site_specific_vars.yml client.yml
 ```
 
+#### Method 2: Packages
+On many operating systems the CVMFS client can be installed through your package manager.
+For details, see the [Getting Started page](https://cvmfs.readthedocs.io/en/stable/cpt-quickstart.html)
+in the documentation.
+
+After installing the client, you will have to configure it.
+For this you can use the CVMFS configuration packages that we provide for clients.
+These packages can be found on the [Releases](https://github.com/eessi/filesystem-layer/releases) page.
+Download the package for your operating system, and install it, e.g.:
+```
+rpm -i cvmfs-config-eessi-*.rpm
+dpkg -i cvmfs-config-eessi-*.deb
+```
+
+Next, you need to make a file `/etc/cvmfs/default.local` manually; this file is used for local settings and
+contains, for instance, the URL to your local proxy and the size of the local cache. As an example, you can put
+the following in this file, which corresponds to not using a proxy and setting the local quota limit to 40000MB:
+```
+CVMFS_HTTP_PROXY=DIRECT
+CVMFS_QUOTA_LIMIT=40000
+```
+For more details about configuring your client, see https://cvmfs.readthedocs.io/en/stable/cpt-configure.html.
+
+Finally, run `cvmfs_config setup` to set up CVMFS.
+
+*Admin note: for building the client configuration packages, see [this section](#building-the-cvmfs-configuration-packages).*
+
 ## Verification and usage
 
 ### Client
@@ -198,3 +227,12 @@ touch /cvmfs/pilot.eessi-hpc.org/testdir/testfile
 sudo cvmfs_server publish pilot.eessi-hpc.org
 ```
 It might take a few minutes, but then the new file should show up at the clients.
+
+
+## Building the CVMFS configuration packages
+
+For each push and pull request to the master branch, packages are automatically built by a Github Action.
+The resulting (unversioned) packages can be found as build artifacts on the page of each run of this action.
+When a new tag is created to mark a versioned release of the repository (e.g. `v1.2.3`, where the `v` is required!),
+the action builds a package with the same version number, creates a release, and stores the packages
+as release assets.

prepare-client-packages.yml

@@ -0,0 +1,45 @@
+# Prepare the input files for the Debian (deb) and Redhat (rpm) packages for CVMFS clients.
+---
+- name: Prepare Deb and RPM packages
+  hosts: localhost
+  vars:
+    package_vendor: EESSI
+    package_maintainer: EESSI <eessi@list.rug.nl>
+    package_description: CVMFS config repository package for EESSI.
+    package_source_dir: "{{ lookup('env', 'GITHUB_WORKSPACE') | default('/tmp', True) }}/package"
+  tasks:
+    - name: Create directory structure for the CVMFS config
+      file:
+       path: "{{ item }}"
+       state: directory
+       mode: 0775
+      with_items:
+      - "{{ package_source_dir }}"
+      - "{{ package_source_dir }}/etc"
+      - "{{ package_source_dir }}/etc/cvmfs"
+      - "{{ package_source_dir }}/etc/cvmfs/keys"
+      - "{{ package_source_dir }}/etc/cvmfs/keys/{{ eessi_cvmfs_config_repo.domain }}"
+      - "{{ package_source_dir }}/etc/cvmfs/config.d"
+      - "{{ package_source_dir }}/etc/cvmfs/default.d"
+
+    - name: Make EESSI CVMFS public key file
+      copy:
+        content: "{{ eessi_cvmfs_config_repo.key.key }}"
+        dest: "{{ package_source_dir }}{{ eessi_cvmfs_config_repo.key.path }}"
+        mode: 0644
+
+    - name: Make EESSI CVMFS configuration file
+      copy:
+        content: |
+          CVMFS_SERVER_URL="{{ eessi_cvmfs_config_repo.urls|join(',') }}"
+          CVMFS_PUBLIC_KEY="{{ eessi_cvmfs_config_repo.key.path }}"
+        dest: "{{ package_source_dir }}/etc/cvmfs/config.d/{{ eessi_cvmfs_config_repo.repository.repository }}.conf"
+        mode: 0644
+
+    - name: Make EESSI CVMFS default configuration file
+      copy:
+        content: |
+          CVMFS_CONFIG_REPOSITORY="{{ eessi_cvmfs_config_repo.repository.repository }}"
+          CVMFS_DEFAULT_DOMAIN="{{ eessi_cvmfs_config_repo.domain }}"
+        dest: "{{ package_source_dir }}/etc/cvmfs/default.d/80-eessi-cvmfs.conf"
+        mode: 0644