feat(components): add submodules and tests

available components' states:

* api
* config
* controller
* keepproxy
* keepstore
* keepweb
* repo
* shell
* websocket
* workbench
* workbench2

added documentation and tests

.rubocop.yml

@@ -6,6 +6,4 @@ Layout/LineLength:
   # Increase from default of `80`
   # Based on https://github.com/PyCQA/flake8-bugbear#opinionated-warnings (`B950`)
   Max: 88
-
 # Any offenses that should be fixed, e.g. collected via. `rubocop --auto-gen-config`
-

.travis.yml

@@ -55,56 +55,12 @@ jobs:
                    @commitlint/travis-cli
         - commitlint-travis
 
-    ## Define the rest of the matrix based on Kitchen testing
-    # Make sure the instances listed below match up with
-    # the `platforms` defined in `kitchen.yml`
-    # NOTE: Please try to select up to six instances that add some meaningful
-    #       testing of the formula's behaviour. If possible, try to refrain from
-    #       the classical "chosing all the instances because I want to test on
-    #       another/all distro/s" trap: it will just add time to the testing (see
-    #       the discussion on #121).  As an example, the set chosen below covers
-    #       the most used distros families, systemd and non-systemd and the latest
-    #       three supported Saltstack versions with python2 and 3.
-    #       As for `kitchen.yml`, that should still contain all of the platforms,
-    #       to allow for comprehensive local testing
-    #       Ref: https://github.com/saltstack-formulas/template-formula/issues/118
-    #       Ref: https://github.com/saltstack-formulas/template-formula/issues/121
-    - env: INSTANCE=default-debian-10-master-py3
-    # - env: INSTANCE=default-ubuntu-1804-master-py3
-    - env: INSTANCE=default-centos-8-master-py3
-    # - env: INSTANCE=default-fedora-31-master-py3
-    # - env: INSTANCE=default-opensuse-leap-151-master-py3
-    # - env: INSTANCE=default-amazonlinux-2-master-py3
-    # - env: INSTANCE=default-arch-base-latest-master-py2
-    # - env: INSTANCE=default-debian-10-3000-1-py3
-    - env: INSTANCE=default-ubuntu-1804-3000-1-py3
-    # - env: INSTANCE=default-centos-8-3000-1-py3
-    # - env: INSTANCE=default-fedora-31-3000-1-py3
-    - env: INSTANCE=default-opensuse-leap-151-3000-1-py3
-    - env: INSTANCE=default-amazonlinux-2-3000-1-py3
-    # - env: INSTANCE=default-arch-base-latest-3000-1-py2
-    # - env: INSTANCE=default-debian-10-2019-2-py3
-    # - env: INSTANCE=default-debian-9-2019-2-py3
-    # - env: INSTANCE=default-ubuntu-1804-2019-2-py3
-    # - env: INSTANCE=default-ubuntu-1604-2019-2-py3
-    # - env: INSTANCE=default-centos-8-2019-2-py3
+    - env: INSTANCE=api-debian-10-3000-1-py3
+    - env: INSTANCE=workbench-debian-10-3000-1-py3
+    - env: INSTANCE=shell-debian-10-3000-1-py3
+    - env: INSTANCE=keepstore-debian-10-3000-1-py3
+    # - env: INSTANCE=default-ubuntu-1804-3000-1-py3
     # - env: INSTANCE=default-centos-7-2019-2-py3
-    - env: INSTANCE=default-fedora-31-2019-2-py3
-    # - env: INSTANCE=default-opensuse-leap-151-2019-2-py3
-    # - env: INSTANCE=default-amazonlinux-2-2019-2-py3
-    # - env: INSTANCE=default-ubuntu-1804-2019-2-py2
-    # - env: INSTANCE=default-amazonlinux-1-2019-2-py2
-    - env: INSTANCE=default-arch-base-latest-2019-2-py2
-    # - env: INSTANCE=default-centos-7-2018-3-py3
-    # - env: INSTANCE=default-fedora-30-2018-3-py3
-    # - env: INSTANCE=default-debian-9-2018-3-py2
-    # - env: INSTANCE=default-debian-8-2018-3-py2
-    # - env: INSTANCE=default-ubuntu-1804-2018-3-py2
-    # - env: INSTANCE=default-ubuntu-1604-2018-3-py2
-    - env: INSTANCE=centos6-centos-6-2018-3-py2
-    # - env: INSTANCE=default-opensuse-leap-151-2018-3-py2
-    # - env: INSTANCE=default-amazonlinux-1-2018-3-py2
-    # - env: INSTANCE=default-arch-base-latest-2018-3-py2
 
     ## Define the release stage that runs `semantic-release`
     - stage: 'release'

.yamllint

@@ -11,6 +11,7 @@ extends: default
 ignore: |
   node_modules/
   test/**/states/**/*.sls
+  test/**/pillar/**/*.sls
   .kitchen/
 
 yaml-files:
@@ -30,7 +31,7 @@ rules:
   line-length:
     # Increase from default of `80`
     # Based on https://github.com/PyCQA/flake8-bugbear#opinionated-warnings (`B950`)
-    max: 88
+    max: 100
   octal-values:
     forbid-implicit-octal: true
     forbid-explicit-octal: true

FORMULA

@@ -1,9 +1,9 @@
 name: arvados
-os: Debian, Ubuntu, Raspbian, RedHat, Fedora, CentOS, Amazon, Suse, openSUSE, Gentoo, Funtoo, Arch, Manjaro, Alpine, FreeBSD, OpenBSD, Solaris, SmartOS, Windows, MacOS
-os_family: Debian, RedHat, Suse, Gentoo, Arch, Alpine, FreeBSD, OpenBSD, Solaris, Windows, MacOS
-version: 1.0.0
+os: Debian, Ubuntu, RedHat, CentOS
+os_family: Debian, RedHat
+version: 0.1.0
 release: 1
-minimum_version: 2017.7
+minimum_version: 3000.1
 summary: arvados formula
-description: Formula to use as a template for other formulas
+description: Formula to install and configure an Arvados cluster
 top_level_dir: arvados

arvados/api/clean.sls

@@ -0,0 +1,6 @@
+# -*- coding: utf-8 -*-
+# vim: ft=sls
+
+include:
+  - .service.dead
+  - .package.clean

arvados/api/init.sls

@@ -0,0 +1,7 @@
+# -*- coding: utf-8 -*-
+# vim: ft=sls
+
+include:
+  - .package
+  - ..config
+  - .service

arvados/api/package/clean.sls

@@ -0,0 +1,28 @@
+# -*- coding: utf-8 -*-
+# vim: ft=sls
+
+{#- Get the `tplroot` from `tpldir` #}
+{%- set tplroot = tpldir.split('/')[0] %}
+{%- from tplroot ~ "/map.jinja" import arvados with context %}
+
+{% for gm in arvados.api.gem.name %}
+arvados-api-package-clean-gem-{{ gm }}-removed:
+  gem.removed:
+    - name: {{ gm }}
+    - require_in:
+      - pkg: arvados-api-package-clean-gems-deps-pkg-removed
+{% endfor %}
+
+arvados-api-package-clean-gems-deps-pkg-removed:
+  pkg.removed:
+    - pkgs: {{ arvados.ruby.gems_deps | json }}
+    - only_if: {{ arvados.ruby.manage_gems_deps }}
+
+arvados-api-package-clean-pkg-removed:
+  pkg.removed:
+    - pkgs: {{ arvados.api.pkg.name | json }}
+
+arvados-api-package-clean-ruby-pkg-removed:
+  pkg.removed:
+    - name: {{ arvados.ruby.pkg }}
+    - only_if: {{ arvados.ruby.manage_ruby }}

arvados/api/package/install.sls

@@ -0,0 +1,37 @@
+# -*- coding: utf-8 -*-
+# vim: ft=sls
+
+{#- Get the `tplroot` from `tpldir` #}
+{%- set tplroot = tpldir.split('/')[0] %}
+{%- set sls_config_file = tplroot ~ '.config.file' %}
+{%- from tplroot ~ "/map.jinja" import arvados with context %}
+
+# The API server requires a valid config BEFORE installing...
+include:
+  - {{ sls_config_file }}
+
+arvados-api-package-install-ruby-pkg-installed:
+  pkg.installed:
+    - name: {{ arvados.ruby.pkg }}
+    - only_if: {{ arvados.ruby.manage_ruby }}
+
+arvados-api-package-install-gems-deps-pkg-installed:
+  pkg.installed:
+    - pkgs: {{ arvados.ruby.gems_deps | json }}
+    - only_if: {{ arvados.ruby.manage_gems_deps }}
+
+{% for gm in arvados.api.gem.name %}
+arvados-api-package-install-gem-{{ gm }}-installed:
+  gem.installed:
+    - name: {{ gm }}
+    - require:
+      - pkg: arvados-api-package-install-gems-deps-pkg-installed
+    - require_in:
+      - pkg: arvados-api-package-install-pkg-installed
+{% endfor %}
+
+arvados-api-package-install-pkg-installed:
+  pkg.installed:
+    - pkgs: {{ arvados.api.pkg.name | json }}
+    - require:
+      - sls: {{ sls_config_file }}

arvados/service/clean.sls → arvados/api/service/clean.sls

@@ -5,7 +5,9 @@
 {%- set tplroot = tpldir.split('/')[0] %}
 {%- from tplroot ~ "/map.jinja" import arvados with context %}
 
-arvados-service-clean-service-dead:
+arvados-api-service-clean-service-dead:
   service.dead:
     - name: {{ arvados.service.name }}
     - enable: False
+    - require_in:
+      - pkg: arvados-api-package-clean-pkg-removed

arvados/service/running.sls → arvados/api/service/running.sls

@@ -7,11 +7,14 @@
 {%- from tplroot ~ "/map.jinja" import arvados with context %}
 
 include:
+  - ..package
   - {{ sls_config_file }}
 
-arvados-service-running-service-running:
+arvados-api-service-running-service-running:
   service.running:
-    - name: {{ arvados.service.name }}
+    - name: {{ arvados.api.service.name }}
     - enable: True
     - watch:
       - sls: {{ sls_config_file }}
+    - require:
+      - pkg: arvados-api-package-install-pkg-installed

arvados/clean.sls

@@ -2,7 +2,13 @@
 # vim: ft=sls
 
 include:
-  - .subcomponent.clean
-  - .service.clean
   - .config.clean
-  - .package.clean
+  #- .api.clean
+  #- .keepstore.clean
+  #- .websocket.clean
+  #- .keepweb.clean
+  #- .workbench2.clean
+  #- .keepproxy.clean
+  - .shell.clean
+  #- .workbench.clean
+  - .repo.clean

arvados/config/clean.sls

@@ -3,14 +3,8 @@
 
 {#- Get the `tplroot` from `tpldir` #}
 {%- set tplroot = tpldir.split('/')[0] %}
-{%- set sls_service_clean = tplroot ~ '.service.clean' %}
 {%- from tplroot ~ "/map.jinja" import arvados with context %}
 
-include:
-  - {{ sls_service_clean }}
-
 arvados-config-clean-file-absent:
   file.absent:
-    - name: {{ arvados.config }}
-    - require:
-      - sls: {{ sls_service_clean }}
+    - name: {{ arvados.config.file }}

arvados/config/file.sls

@@ -3,26 +3,20 @@
 
 {#- Get the `tplroot` from `tpldir` #}
 {%- set tplroot = tpldir.split('/')[0] %}
-{%- set sls_package_install = tplroot ~ '.package.install' %}
 {%- from tplroot ~ "/map.jinja" import arvados with context %}
 {%- from tplroot ~ "/libtofs.jinja" import files_switch with context %}
 
-include:
-  - {{ sls_package_install }}
-
 arvados-config-file-file-managed:
   file.managed:
-    - name: {{ arvados.config }}
-    - source: {{ files_switch(['example.tmpl'],
+    - name: {{ arvados.config.file }}
+    - source: {{ files_switch(['config.tmpl', 'config.tmpl.jinja'],
                               lookup='arvados-config-file-file-managed'
                  )
               }}
-    - mode: 644
-    - user: root
-    - group: {{ arvados.rootgroup }}
+    - mode: {{ arvados.config.mode }}
+    - user: {{ arvados.config.user }}
+    - group: {{ arvados.config.group }}
     - makedirs: True
     - template: jinja
-    - require:
-      - sls: {{ sls_package_install }}
     - context:
         arvados: {{ arvados | json }}

arvados/controller/clean.sls

@@ -0,0 +1,6 @@
+# -*- coding: utf-8 -*-
+# vim: ft=sls
+
+include:
+  - .service.dead
+  - .package.clean

arvados/controller/init.sls

@@ -0,0 +1,7 @@
+# -*- coding: utf-8 -*-
+# vim: ft=sls
+
+include:
+  - .package
+  - ..config
+  - .service

arvados/controller/package/clean.sls

@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+# vim: ft=sls
+
+{#- Get the `tplroot` from `tpldir` #}
+{%- set tplroot = tpldir.split('/')[0] %}
+{%- from tplroot ~ "/map.jinja" import arvados with context %}
+
+arvados-controller-package-clean-pkg-removed:
+  pkg.removed:
+    - name: {{ arvados.controller.pkg.name }}

arvados/subcomponent/config/init.sls → arvados/controller/package/init.sls

@@ -2,4 +2,4 @@
 # vim: ft=sls
 
 include:
-  - .file
+  - .install

arvados/controller/package/install.sls

@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+# vim: ft=sls
+
+{#- Get the `tplroot` from `tpldir` #}
+{%- set tplroot = tpldir.split('/')[0] %}
+{%- from tplroot ~ "/map.jinja" import arvados with context %}
+
+arvados-controller-package-install-pkg-installed:
+  pkg.installed:
+    - name: {{ arvados.controller.pkg.name }}

arvados/controller/service/clean.sls

@@ -0,0 +1,13 @@
+# -*- coding: utf-8 -*-
+# vim: ft=sls
+
+{#- Get the `tplroot` from `tpldir` #}
+{%- set tplroot = tpldir.split('/')[0] %}
+{%- from tplroot ~ "/map.jinja" import arvados with context %}
+
+arvados-controller-service-clean-service-dead:
+  service.dead:
+    - name: {{ arvados.service.name }}
+    - enable: False
+    - require_in:
+      - pkg: arvados-controller-package-clean-pkg-removed

arvados/subcomponent/init.sls → arvados/controller/service/init.sls

@@ -2,4 +2,4 @@
 # vim: ft=sls
 
 include:
-  - .config
+  - .running

arvados/controller/service/running.sls

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# vim: ft=sls
+
+{#- Get the `tplroot` from `tpldir` #}
+{%- set tplroot = tpldir.split('/')[0] %}
+{%- set sls_config_file = tplroot ~ '.config.file' %}
+{%- from tplroot ~ "/map.jinja" import arvados with context %}
+
+include:
+  - ..package
+  - {{ sls_config_file }}
+
+arvados-controller-service-running-service-running:
+  service.running:
+    - name: {{ arvados.controller.service.name }}
+    - enable: True
+    - watch:
+      - sls: {{ sls_config_file }}
+    - require:
+      - pkg: arvados-controller-package-install-pkg-installed