feat(archives): support for archives file format

noelmcloughlin · noelmcloughlin · commit 1f86f4a27ca7 · 2019-06-15T01:52:59.000+01:00
BREAKING CHANGE: the parameter `pkg` is now a dictionary. References
 to `prometheus.pkg` should be changed to `prometheus.pkg.name`.
diff --git a/docs/README.rst b/docs/README.rst
@@ -48,11 +48,21 @@ This installs the prometheus package,
 manages the prometheus configuration file and then
 starts the associated prometheus service.
 
+``prometheus.archive``
+^^^^^^^^^^^^^^^^^^^^
+
+This state will install the prometheus from archive file only.
+
 ``prometheus.package``
 ^^^^^^^^^^^^^^^^^^^^
 
 This state will install the prometheus package only.
 
+``prometheus.package.repo``
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+This state will install the prometheus package only.
+
 ``prometheus.config``
 ^^^^^^^^^^^^^^^^^^^
 
@@ -92,6 +102,16 @@ dependency on ``prometheus.service.clean`` via include list.
 This state will remove the prometheus package and has a depency on
 ``prometheus.config.clean`` via include list.
 
+``prometheus.package.archive.clean``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+This state will uninstall the prometheus archive-extracted directory only.
+
+``prometheus.package.repo.clean``
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+This state will uninstall the prometheus upstream package repository only.
+
 ``prometheus.exporters``
 ^^^^^^^^^^^^^^^^^^^^^^^^
 
diff --git a/pillar.example b/pillar.example
@@ -2,8 +2,14 @@
 # vim: ft=yaml
 ---
 prometheus:
-  pkg: prometheus
+  pkg:
+    name: prometheus
+    archive:
+      source: https://github.com/prometheus/prometheus/releases/download/v2.10.0/prometheus-2.10.0.linux-amd64.tar.gz
+      source_hash: f4233783826f18606b79e5cef0686e4a9c2030146a3c7ce134f0add09f5adcb7
+      enforce_toplevel: False
   config_file: /etc/prometheus/prometheus.yml
+  environ_file: /etc/default/prometheus.sh
   service:
     name: prometheus
     args:
@@ -33,6 +39,8 @@ prometheus:
     #     - 'example_alt.tmpl.jinja'
 
   # Pillar-based config
+  environ:
+    - 'export PATH=${PATH}:/opt/prometheus-2.10.0.linux-amd64'
   config:
     # my global config
     global:
diff --git a/prometheus/archive/clean.sls b/prometheus/archive/clean.sls
@@ -0,0 +1,11 @@
+# -*- coding: utf-8 -*-
+# vim: ft=sls
+
+{#- Get the `tplroot` from `tpldir` #}
+{%- set tplroot = tpldir.split('/')[0] %}
+{%- from tplroot ~ "/map.jinja" import prometheus with context %}
+
+prometheus-cli-package-archive-clean-file-absent:
+  file.absent:
+    - names:
+      - {{ prometheus.base_dir }}
diff --git a/prometheus/archive/init.sls b/prometheus/archive/init.sls
@@ -0,0 +1,5 @@
+#.-*- coding: utf-8 -*-
+# vim: ft=sls
+
+include:
+  - .install
diff --git a/prometheus/archive/install.sls b/prometheus/archive/install.sls
@@ -0,0 +1,23 @@
+# -*- coding: utf-8 -*-
+# vim: ft=sls
+
+{#- Get the `tplroot` from `tpldir` #}
+{%- set tplroot = tpldir.split('/')[0] %}
+{%- from tplroot ~ "/map.jinja" import prometheus with context %}
+{%- from tplroot ~ "/jinja/macros.jinja" import format_kwargs with context %}
+
+prometheus-package-archive-install-file-directory:
+  file.directory:
+    - name: {{ prometheus.pkg.archive.name }}
+    - makedirs: True
+    - require_in:
+      - archive: prometheus-package-archive-install-archive-extracted
+
+prometheus-package-archive-install-archive-extracted:
+  archive.extracted:
+    {{- format_kwargs(prometheus.pkg.archive) }}
+    - retry:
+        attempts: 3
+        until: True
+        interval: 60
+        splay: 10
diff --git a/prometheus/clean.sls b/prometheus/clean.sls
@@ -4,4 +4,5 @@
 include:
   - .service.clean
   - .config.clean
+  - .archive.clean
   - .package.clean
diff --git a/prometheus/config/clean.sls b/prometheus/config/clean.sls
@@ -11,7 +11,9 @@ include:
 
 prometheus-config-clean-file-absent:
   file.absent:
-    - name: {{ prometheus.config_file }}
+    - names:
+       - {{ prometheus.config_file }}
+       - {{ prometheus.environ_file }}
     - require:
       - sls: {{ sls_service_clean }}
 
diff --git a/prometheus/config/environ.sls b/prometheus/config/environ.sls
@@ -0,0 +1,28 @@
+# -*- coding: utf-8 -*-
+# vim: ft=sls
+
+{#- Get the `tplroot` from `tpldir` #}
+{%- set tplroot = tpldir.split('/')[0] %}
+{%- from tplroot ~ "/map.jinja" import prometheus with context %}
+{%- set sls_archive_install = tplroot ~ '.archive.install' %}
+{%- from tplroot ~ "/libtofs.jinja" import files_switch with context %}
+
+include:
+  - {{ sls_archive_install }}
+
+prometheus-config-file-file-managed-environ_file:
+  file.managed:
+    - name: {{ prometheus.environ_file }}
+    - source: {{ files_switch(['prometheus.sh.jinja'],
+                              lookup='prometheus-config-file-file-managed-environ_file'
+                 )
+              }}
+    - mode: 644
+    - user: root
+    - group: {{ prometheus.rootgroup }}
+    - makedirs: True
+    - template: jinja
+    - context:
+        prometheus: {{ prometheus|json }}
+    - require:
+      - sls: {{ sls_archive_install }}
diff --git a/prometheus/config/file.sls b/prometheus/config/file.sls
@@ -3,18 +3,24 @@
 
 {#- Get the `tplroot` from `tpldir` #}
 {%- set tplroot = tpldir.split('/')[0] %}
-{%- set sls_package_install = tplroot ~ '.package.install' %}
 {%- from tplroot ~ "/map.jinja" import prometheus with context %}
 {%- from tplroot ~ "/libtofs.jinja" import files_switch with context %}
 
+{%- if 'config' in prometheus and prometheus.config %}
+    {%- if prometheus.pkg.use_upstream_archive %}
+        {%- set sls_package_install = tplroot ~ '.archive.install' %}
+    {%- else %}
+        {%- set sls_package_install = tplroot ~ '.package.install' %}
+    {%- endif %}
+
 include:
   - {{ sls_package_install }}
 
-prometheus-config-file-file-managed:
+prometheus-config-file-file-managed-config_file:
   file.managed:
     - name: {{ prometheus.config_file }}
     - source: {{ files_switch(['prometheus.yml.jinja'],
-                              lookup='prometheus-config-file-file-managed'
+                              lookup='prometheus-config-file-file-managed-config_file'
                  )
               }}
     - mode: 644
@@ -26,3 +32,5 @@ prometheus-config-file-file-managed:
         config: {{ prometheus.config|json }}
     - require:
       - sls: {{ sls_package_install }}
+
+{%- endif %}
diff --git a/prometheus/config/init.sls b/prometheus/config/init.sls
@@ -4,3 +4,4 @@
 include:
   - .args
   - .file
+  - .environ
diff --git a/prometheus/defaults.yaml b/prometheus/defaults.yaml
@@ -2,15 +2,32 @@
 # vim: ft=yaml
 ---
 prometheus:
-  pkg: prometheus
+  version: '2.10.0'
+  pkg:
+    name: prometheus
+    binary: prometheus
+    use_upstream_archive: False
+    archive:
+      name: /opt
+      uri: https://github.com/prometheus/prometheus/releases/download/
+      source: None
+      # linux amd64 source hash
+      source_hash: f4233783826f18606b79e5cef0686e4a9c2030146a3c7ce134f0add09f5adcb7
+      trim_output: True
+      enforce_toplevel: True
+      archive_format: tar.gz
   rootgroup: root
+  kernel: {{ grains.kernel | lower }}
   config_file: /etc/prometheus/prometheus.yml
   config: {}
+  environ_file: /etc/default/prometheus.sh
+  environ: []
   service:
     name: prometheus
     user: prometheus
     group: prometheus
   exporters:
     node:
-      pkg: prometheus-node-exporter
+      pkg:
+        name: prometheus-node-exporter
       service: prometheus-node-exporter
diff --git a/prometheus/exporters/node/clean.sls b/prometheus/exporters/node/clean.sls
@@ -12,7 +12,7 @@ prometheus-exporters-node-service-dead:
 
 prometheus-exporters-node-pkg-removed:
   pkg.removed:
-    - name: {{ prometheus.exporters.node.pkg }}
+    - name: {{ prometheus.exporters.node.pkg.name }}
     - require:
       - service: prometheus-exporters-node-service-dead
 
diff --git a/prometheus/exporters/node/init.sls b/prometheus/exporters/node/init.sls
@@ -8,7 +8,7 @@
 
 prometheus-exporters-node-pkg-installed:
   pkg.installed:
-    - name: {{ prometheus.exporters.node.pkg }}
+    - name: {{ prometheus.exporters.node.pkg.name }}
 
 {%- if 'args' in prometheus.exporters.node %}
 {%-   set args = prometheus.exporters.node.get('args', {}) -%}
diff --git a/prometheus/files/default/prometheus.sh.jinja b/prometheus/files/default/prometheus.sh.jinja
@@ -0,0 +1,10 @@
+########################################################################
+# File managed by Salt at <{{ source }}>.
+# Your changes may be overwritten.
+########################################################################
+
+{%- if prometheus.environ %}
+    {%- for item in prometheus.environ %}
+{{ item }}
+    {%- endfor %}
+{%- endif %}
diff --git a/prometheus/files/default/prometheus.yml.jinja b/prometheus/files/default/prometheus.yml.jinja
@@ -1,6 +1,6 @@
 ########################################################################
 # File managed by Salt at <{{ source }}>.
-# Your changes will be overwritten.
+# Your changes may be overwritten.
 ########################################################################
 
 {{ config|yaml(False) }}
diff --git a/prometheus/init.sls b/prometheus/init.sls
@@ -1,7 +1,11 @@
 # -*- coding: utf-8 -*-
 # vim: ft=sls
 
+{#- Get the `tplroot` from `tpldir` #}
+{%- set tplroot = tpldir.split('/')[0] %}
+{%- from tplroot ~ "/map.jinja" import prometheus with context %}
+
 include:
-  - .package
+  - {{ '.archive' if prometheus.pkg.use_upstream_archive else '.package' }}
   - .config
   - .service
diff --git a/prometheus/jinja/macros.jinja b/prometheus/jinja/macros.jinja
@@ -0,0 +1,12 @@
+# -*- coding: utf-8 -*-
+# vim: ft=jinja
+#
+# Collection of common macros
+
+{%- macro format_kwargs(kwarg) -%}
+  {%- filter indent(4) %}
+    {%- for k, v in kwarg|dictsort() %}
+- {{ k }}: {{ v }}
+    {%- endfor %}
+  {%- endfilter %}
+{%- endmacro %}
diff --git a/prometheus/map.jinja b/prometheus/map.jinja
@@ -7,14 +7,17 @@
 {%- import_yaml tplroot ~ "/defaults.yaml" as default_settings %}
 {%- import_yaml tplroot ~ "/osfamilymap.yaml" as osfamilymap %}
 {%- import_yaml tplroot ~ "/osmap.yaml" as osmap %}
+{%- import_yaml tplroot ~ "/osarchmap.yaml" as osarchmap %}
 {%- import_yaml tplroot ~ "/osfingermap.yaml" as osfingermap %}
 
 {%- set defaults = salt['grains.filter_by'](default_settings,
     default='prometheus',
     merge=salt['grains.filter_by'](osfamilymap, grain='os_family',
       merge=salt['grains.filter_by'](osmap, grain='os',
         merge=salt['grains.filter_by'](osfingermap, grain='osfinger',
-          merge=salt['pillar.get']('prometheus:lookup', default={})
+          merge=salt['grains.filter_by'](osarchmap, grain='osarch',
+            merge=salt['pillar.get']('prometheus:lookup', default={})
+          )
         )
       )
     )
@@ -23,6 +26,18 @@
 {#- Merge the prometheus pillar #}
 {%- set prometheus = salt['pillar.get']('prometheus', default=defaults, merge=True) %}
 
+{#- Update archive details #}
+{%- if prometheus.pkg.use_upstream_archive %}
+    {%- set name = 'prometheus-%s.%s-%s'|format(prometheus.version, prometheus.kernel, prometheus.arch) %}
+    {%- do prometheus.pkg.archive.update({
+           'source': prometheus.pkg.archive.uri + 'v' + prometheus.version + '/' + name 
+                                                + '.' + prometheus.pkg.archive.archive_format,
+           'archive_format': prometheus.pkg.archive.archive_format.split('.')[0]
+    }) %}
+    {%- do prometheus.update({'base_dir': prometheus.pkg.archive.name + '/' + name}) %}
+    {%- do prometheus.environ.append('export PATH=${PATH}:' + prometheus.pkg.archive.name + '/' + name) %}
+{%- endif %}
+
 {#- Contactenate arguments #}
 {%- macro concat_args(args) %}
 {%-   set args = args|dictsort %}
diff --git a/prometheus/osarchmap.yaml b/prometheus/osarchmap.yaml
@@ -0,0 +1,35 @@
+# -*- coding: utf-8 -*-
+# vim: ft=yaml
+#
+# Setup variables using grains['osarch'] based logic.
+# You just need to add the key:values for an `osarch` that differ
+# from `defaults.yaml` + `os_family.yaml`.
+# Only add an `osarch` which is/will be supported by the formula
+#
+# If you do not need to provide defaults via the `osarch` grain,
+# you will need to provide at least an empty dict in this file, e.g.
+# osarch: {}
+---
+amd64:
+  arch: amd64
+
+x86_64:
+  arch: amd64
+
+386:
+  arch: 386
+
+arm64:
+  arch: arm64
+
+armv6l:
+  arch: armv6l
+
+armv7l:
+  arch: armv6l
+
+ppc64le:
+  arch: ppc64le
+
+s390x:
+  arch: s390x
diff --git a/prometheus/osfamilymap.yaml b/prometheus/osfamilymap.yaml
@@ -35,15 +35,26 @@ FreeBSD:
   config_file: /usr/local/etc/prometheus.yml
   exporters:
     node:
-      pkg: node_exporter
+      pkg:
+        name: node_exporter
       service: node_exporter
 
 OpenBSD:
   rootgroup: wheel
 
 Solaris: {}
 
-Windows: {}
+Windows:
+  pkg:
+    archive:
+      name: C:\\Program Files
+      source: https://github.com/prometheus/prometheus/releases/download/v2.10.0/prometheus-2.10.0.windows-amd64.tar.gz
+      source_hash: eb138082a4d5e4d5b1e3ca838fa508f053474d46bca76e87ab0834f0d8b110db
 
 MacOS:
+  pkg:
+    archive:
+      name: /opt
+      source: https://github.com/prometheus/prometheus/releases/download/v2.10.0/prometheus-2.10.0.darwin-amd64.tar.gz
+      source_hash: 740e36bcacc0c5d4495f5341fcfa8b7e0dc623d12e8b07ac291052ea0a681325
   rootgroup: {{ macos_group | d('') }}
diff --git a/prometheus/package/clean.sls b/prometheus/package/clean.sls
@@ -11,6 +11,6 @@ include:
 
 prometheus-package-clean-pkg-removed:
   pkg.removed:
-    - name: {{ prometheus.pkg }}
+    - name: {{ prometheus.pkg.name }}
     - require:
       - sls: {{ sls_config_clean }}
diff --git a/prometheus/package/install.sls b/prometheus/package/install.sls
diff --git a/prometheus/service/running.sls b/prometheus/service/running.sls
