Merge branch 'release/1.4.0'

.travis.yml

@@ -1,4 +1,6 @@
 sudo: required
+git:
+  depth: false
 services:
 - docker
 install:

debian/changelog

@@ -1,3 +1,12 @@
+edi (1.4.0) bionic; urgency=medium
+
+  [ Matthias Lüscher ]
+  * Added possibility to keep the changelog but remove all the remaining docs.
+  * Added possibility to document the build setup.
+  * Added possibility to document the installed packages.
+
+ -- Matthias Lüscher (Launchpad) <m.luescher@datacomm.ch>  Fri, 27 Mar 2020 23:13:19 +0100
+
 edi (1.3.0) bionic; urgency=medium
 
   [ Matthias Lüscher ]

docs/conf.py

@@ -55,8 +55,8 @@
 # built documents.
 #
 # The short X.Y version.
-version = '1.3.0'
-release = '1.3.0'
+version = '1.4.0'
+release = '1.4.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/config_management/plugins.rst

@@ -207,8 +207,9 @@ The playbook can be fine tuned as follows:
        :code:`{{ edi_project_directory }}/ssh_pub_keys`) will be added to the list of authorized ssh keys of the
        default user.
    * - install_documentation
-     - By default (boolean value :code:`True`) the documentation of every Debian package will get installed.
-       Switch this value to :code:`False` if you want to deploy an image with a minimal footprint.
+     - By default (value :code:`full`) the documentation of every Debian package will get installed.
+       Switch this value to :code:`minimal` if you want to deploy an image with a minimal footprint.
+       Switch this value to :code:`changelog` if you want to minimize the footprint but keep the changelog of all packages.
    * - translations_filter
      - By default all translations contained in Debian packages will get installed (empty filter: :code:`""`).
        To reduce the footprint of the resulting artifacts the number of installed languages can be limited.
@@ -308,6 +309,16 @@ The playbook can be fine tuned as follows:
        (boolean value :code:`True`). If you would like to keep the same proxy settings switch this value to
        :code:`False`. When set to :code:`True`, the proxy settings can be fine tuned according to the table
        below.
+   * - document_build_setup
+     - To document the build setup of the artifact within the artifact set this value to :code:`True`.
+       As a result the file :code:`/usr/share/doc/edi/build.yml` will be generated. By default this feature is switched
+       off (boolean value :code:`False`).
+   * - document_packages
+     - To document the packages of the artifact within the artifact set this value to :code:`True`.
+       As a result the file :code:`/usr/share/doc/edi/packages.yml` will be generated. The generated file will contain a
+       list of all packages including version information. It is a snapshot of the available packages after the artifact
+       build and will not get updated when new packages get installed using :code:`dpkg` or :code:`apt`.
+       By default this feature is switched off (boolean value :code:`False`).
 
 The final proxy settings can be customized as follows:
 

edi/lib/configurationparser.py

@@ -73,6 +73,7 @@ def get_base_dictionary():
     base_dict["edi_host_ftp_proxy"] = proxy_setup.get('ftp_proxy', default='')
     base_dict["edi_host_socks_proxy"] = proxy_setup.get('all_proxy', default='')
     base_dict["edi_host_no_proxy"] = proxy_setup.get('no_proxy', default='')
+    base_dict["edi_edi_version"] = get_edi_version()
     base_dict["edi_lxd_version"] = get_lxd_version()
     return base_dict
 

edi/lib/versionhelpers.py

@@ -26,7 +26,7 @@
 
 # The do_release script will update this version!
 # During launchpad debuild neither the git version nor the package version is available.
-edi_fallback_version = '1.3.0'
+edi_fallback_version = '1.4.0'
 
 
 def get_edi_version():

edi/plugins/playbooks/debian/base_system/main.yml

@@ -7,7 +7,7 @@
     disable_ssh_password_authentication: True
     authorize_current_user: True
     ssh_pub_key_directory: '{{ edi_project_directory }}/ssh_pub_keys'
-    install_documentation: True
+    install_documentation: full
     translations_filter: ""
   roles:
     - role: lxc_interfaces

edi/plugins/playbooks/debian/base_system/roles/apt_setup/defaults/main.yml

@@ -1,6 +1,6 @@
 ---
 
-install_documentation: True
+install_documentation: full
 dpkg_no_documentation: /etc/dpkg/dpkg.cfg.d/01_no_documentation
 translations_filter: ""
 dpkg_translations_filter: /etc/dpkg/dpkg.cfg.d/02_translations_filter

edi/plugins/playbooks/debian/base_system/roles/apt_setup/files/remove_documentation_keep_changelog

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+set -o errexit
+set -o nounset
+
+DPKG_CONTROL_FILE=$1
+
+find /usr/share/doc -depth -type f ! -name copyright ! -name changelog.Debian.gz ! -name changelog.gz |xargs rm || true
+find /usr/share/doc -empty|xargs rmdir || true
+rm -rf /usr/share/man /usr/share/groff /usr/share/info
+rm -rf /usr/share/lintian /usr/share/linda /var/cache/man
+
+cat << EOF > ${DPKG_CONTROL_FILE}
+# save disk space by excluding documentation
+path-exclude /usr/share/doc/*
+# keep copyright files for legal reasons
+path-include /usr/share/doc/*/copyright
+# keep changelog files
+path-include /usr/share/doc/*/changelog.Debian.gz
+path-include /usr/share/doc/*/changelog.gz
+path-exclude /usr/share/man/*
+path-exclude /usr/share/groff/*
+path-exclude /usr/share/info/*
+path-exclude /usr/share/lintian/*
+path-exclude /usr/share/linda/*
+EOF
+
+chmod 644 ${DPKG_CONTROL_FILE}

edi/plugins/playbooks/debian/base_system/roles/apt_setup/tasks/main.yml

@@ -10,7 +10,13 @@
   script: remove_documentation {{ dpkg_no_documentation }}
   args:
     creates: "{{ dpkg_no_documentation }}"
-  when: not install_documentation
+  when: install_documentation == 'minimal' or install_documentation is sameas false
+
+- name: Prevent documentation installation and remove existing documentation but keep changelog.
+  script: remove_documentation_keep_changelog {{ dpkg_no_documentation }}
+  args:
+    creates: "{{ dpkg_no_documentation }}"
+  when: install_documentation == 'changelog'
 
 - name: Install only a subset of translations.
   script: "{{ translations_filter }} {{ dpkg_translations_filter }}"

edi/plugins/playbooks/debian/base_system_cleanup/main.yml

@@ -8,6 +8,8 @@
     target_ftp_proxy: ""
     target_socks_proxy: ""
     target_no_proxy: ""
+    document_build_setup: False
+    document_packages: False
   roles:
     - role: openssh_server_keys
       become: True
@@ -24,4 +26,7 @@
       become: True
       become_user: root
       when: edi_create_distributable_image
+    - role: document_artifact
+      become: True
+      become_user: root
 

edi/plugins/playbooks/debian/base_system_cleanup/roles/document_artifact/defaults/main.yml

@@ -0,0 +1,7 @@
+---
+document_build_setup: False
+document_packages: False
+documentation_directory: /usr/share/doc/edi
+package_documentation_command: "dpkg-query --show --showformat='${Package}:\n    architecture: ${Architecture}\n    version: \"${Version}\"\n    source_package: ${source:Package}\n    status: ${db:Status-Abbrev}\n'"
+package_documentation_file: packages.yml
+build_documentation_file: build.yml

edi/plugins/playbooks/debian/base_system_cleanup/roles/document_artifact/files/get_project_scm_revision

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+set -o nounset
+set -o errexit
+set -o pipefail
+
+print_error_and_exit()
+{
+    local MESSAGE="${1}"
+    >&2 echo -e "Error: ${MESSAGE}"
+    exit 1
+}
+
+if [[ "$#" -ne 1 ]]
+then
+    print_error_and_exit "Missing project directory.\nUsage: ${0} PROJECT_DIRECTORY"
+fi
+
+PROJECT_DIRECTORY=${1}
+
+if [[ ! -d ${PROJECT_DIRECTORY} ]]
+then
+    print_error_and_exit "Invalid project directory ${PROJECT_DIRECTORY}."
+fi
+
+if ! command -v git > /dev/null
+then
+    echo "git-not-available"
+    exit 0
+fi
+
+cd "${PROJECT_DIRECTORY}"
+
+if ! git rev-parse HEAD 2> /dev/null
+then
+    echo "not-a-git-repository"
+    exit 0
+fi

edi/plugins/playbooks/debian/base_system_cleanup/roles/document_artifact/tasks/main.yml

@@ -0,0 +1,32 @@
+---
+- name: Create documentation directory within artifact.
+  file:
+    dest: "{{ documentation_directory }}"
+    state: directory
+  when: document_build_setup or document_packages
+
+- name: Get project configuration scm revision.
+  script: get_project_scm_revision {{ edi_project_directory }}
+  delegate_to: 127.0.0.1
+  register: project_scm_revision
+  changed_when: false
+  become: no
+  when: document_build_setup
+
+- name: Write build information file.
+  template:
+    src: build.yml
+    dest: "{{ documentation_directory }}/{{ build_documentation_file }}"
+  when: document_build_setup
+
+- name: Generate a list of installed packages.
+  command: "{{ package_documentation_command }}"
+  register: installed_packages
+  changed_when: false
+  when: document_packages
+
+- name: Write a file that documents the installed packages.
+  template:
+    src: packages
+    dest: "{{ documentation_directory }}/{{ package_documentation_file }}"
+  when: document_packages

edi/plugins/playbooks/debian/base_system_cleanup/roles/document_artifact/templates/build.yml

@@ -0,0 +1,4 @@
+configuration_name: "{{ edi_configuration_name }}"
+project_scm_revision: "{{ project_scm_revision.stdout.strip('\n') }}"
+edi_version: "{{ edi_edi_version }}"
+ansible_version: "{{ ansible_version.full }}"

edi/plugins/playbooks/debian/base_system_cleanup/roles/document_artifact/templates/packages

@@ -0,0 +1 @@
+{{ installed_packages.stdout }}

setup.py

@@ -38,7 +38,7 @@
 setup(
     name='edi',
 
-    version='1.3.0',
+    version='1.4.0',
 
     description='Embedded Development Infrastructure - edi',
     long_description=long_description,

tests/conftest.py

@@ -93,7 +93,7 @@ def datadir(tmpdir, request):
         folder:             skip
         mountpoint:         me
     workspace:
-        folder:             work
+        folder:             edi_marker_work
         mountpoint:         mywork
 
 bootstrap:
@@ -173,7 +173,7 @@ def datadir(tmpdir, request):
 sample_system_file = """
 shared_folders:
     other_folder:
-        folder:             valid_folder
+        folder:             edi_marker_valid_folder
     skip_me:
         skip:               True
 

tests/lib/test_commandrunner.py

@@ -37,7 +37,6 @@ def intercept_command_run(*popenargs, **kwargs):
                 print(f.read())
             return subprocess.run(*popenargs, **kwargs)
         else:
-            print("passthrough")
             return subprocess.run(*popenargs, **kwargs)
 
     monkeypatch.setattr(mockablerun, 'run_mockable', intercept_command_run)

tests/lib/test_configurationparser.py

@@ -97,15 +97,15 @@ def test_shared_folders(config_files):
         name, content, dict = shared_folders[0]
         assert name == 'other_folder'
         assert content.get('mountpoint') == 'target_mountpoint'
-        assert content.get('folder') == 'valid_folder'  # merge result
+        assert content.get('folder') == 'edi_marker_valid_folder'  # merge result
         assert dict.get('edi_current_user_host_home_directory')
         assert dict.get('edi_current_user_target_home_directory') == '/foo/bar'
 
         # second element
         name, content, dict = shared_folders[1]
         assert name == 'workspace'
         assert content.get('mountpoint') == 'mywork'
-        assert content.get('folder') == 'work'
+        assert content.get('folder') == 'edi_marker_work'
         assert dict.get('edi_current_user_target_home_directory').startswith('/home/')