Summarize packages for the Linux platform

Before, only the Linux packages of bareos-15.2 have been shown.
Now, the tables try to show what packages are available from 12.4 to
16.2.

manuals/en/main/Makefile

@@ -32,7 +32,7 @@ GENERATED = autogenerated/bareos-dir-*.tex \
 	autogenerated/bareos-fd-*.tex \
 	autogenerated/bconsole-*.tex \
 	autogenerated/bareos-tray-monitor-*.tex \
-	autogenerated/bareos-15.2-packages-table-*.tex
+	autogenerated/bareos-packages-table-*.tex
 
 LOG = bareos-manual-main-reference.log
 
@@ -139,8 +139,8 @@ autogenerated/bareos-tray-monitor-*.tex: autogenerated/bareos-tray-monitor-confi
 	# generates all tray-monitor resource files
 	$(GENERATED_RESOURCE_DESCRIPTIONS) autogenerated/bareos-tray-monitor-config-schema.json
 
-autogenerated/bareos-15.2-packages-table-*.tex:
-	$(GENERATED_PACKAGE_INFO) -o autogenerated/ data/bareos-15.2-packages.json
+autogenerated/bareos-packages-table-*.tex:
+	$(GENERATED_PACKAGE_INFO) -o autogenerated/ data/bareos-12.4-packages.json data/bareos-13.2-packages.json data/bareos-14.2-packages.json data/bareos-15.2-packages.json data/bareos-16.2-packages.json
 
 check: bareos-manual-main-reference.pdf
 	@# hyperlink: multiline: pdfTeX warning .* has been reference$d but does not exist

manuals/en/main/supportedoses.tex

@@ -105,22 +105,30 @@ \section{}
 \subsection{Packages for the different Linux platforms}
 \label{sec:packages}
 
-The following tables show what packages are included for the different platforms with Bareos on \url{http://download.bareos.org/bareos/release/15.2/}.
-Bareos tries to provide all packages for all platforms.
+The following tables summarize what packages are available for the different Linux platforms.
 
-On extra packages, it depends if the distribution contains the required dependencies.
+This information is generated based on \url{http://download.bareos.com/bareos/release/}.
+In most cases this is identical to the packages provided by \url{http://download.bareos.org/bareos/release/}.
+Only if a package have been added later in a maintenance release, these information may differ.
+
+Distributions that are no longer relevant are left out.
+However, you might still find the packages on our download servers.
+
+Bareos tries to provide all packages for all current platforms.
+For extra packages, it depends if the distribution contains the required dependencies.
 
 Packages names not containing the word \name{bareos} are required packages where we decided to include them ourselves.
 
 {
     \small
-    \input{autogenerated/bareos-15.2-packages-table-redhat.tex}
-    \input{autogenerated/bareos-15.2-packages-table-suse.tex}
-    \input{autogenerated/bareos-15.2-packages-table-debian.tex}
+    \input{autogenerated/bareos-packages-table-redhat.tex}
+    \input{autogenerated/bareos-packages-table-fedora.tex}
+    \input{autogenerated/bareos-packages-table-suse.tex}
+    \input{autogenerated/bareos-packages-table-debian.tex}
+    \input{autogenerated/bareos-packages-table-ubuntu.tex}
 }
 
 
-
 \input{operating-system-univention-corporate-server.tex}
 
 

manuals/scripts/generate-bareos-package-info.py

@@ -2,34 +2,26 @@
 
 # -*- coding: utf-8 -*-
 
-"""ospi-client: performs operation for opsi clients on opsi server via JSON-RPC."""
+"""
+Generate-bareos-package-info: generate latex tables from a json data structure.
+
+Use obs-project-binary-packages-to-json.py to create the json files.
+"""
 
 __author__ = "Joerg Steffens"
-__copyright__ = "2015"
+__copyright__ = "2015-2016"
 __license__ = "AGPL"
-__version__ = "0.1"
+__version__ = "0.2"
 __email__ = "joerg.steffens@bareos.com"
 
 import argparse
-#from   debian.debfile import DebFile
 import logging
 import json
 import os
 from   pprint import pprint
 import re
-#import rpm
-
-# JSON structure:
-# 
-# 'packages': {
-#   package_name: [ dist1, dist2, ... ],
-#   ...
-# },
-# 'distributions': {
-#   distribution_name: [ package1, package2, ... ],
-#   ...
-# }
-#
+
+
 
 class LatexTable:
     def __init__(self, columns, rows):
@@ -56,14 +48,13 @@ def __getHeaderColumns(self):
             prefix_new=i.split('_',1)[0].replace('_', ' ')
             if prefix_new != prefix:
                 if prefixcount:
-                    #result+=" & " + prefix + str(prefixcount)
                     result+='\\multicolumn{%i}{ c|}{%s} &\n' % (prefixcount, prefix)
                 prefix = prefix_new
                 prefixcount=1
             else:
                 prefixcount+=1
-            version = i.rsplit('_',1)[1]
-            version_header+=" & " + version.replace('_', '')
+            version = i.split('_',1)[1]
+            version_header+=" & " + version.replace('_', '').replace('Leap', '').replace('SP','sp')
         if prefixcount:
             result+='\\multicolumn{%i}{ c }{%s}\n' % (prefixcount, prefix)
         result+="\\\\ \n"
@@ -90,46 +81,118 @@ def get(self):
         result += self.__getFooter()
         return result
 
-#class OverviewTable:
-def generate_overview_table(data, dist_include):
-    dist_exclude_pattern = re.compile("win_cross|windows")
-    package_exclude_pattern = re.compile("php-ZendFramework2-.*|mhvtl-kmp-.*|mingw.*|.*-debug.*")
-    packages_excluded = []
-    distributions = []
-    for dist in sorted(data['distributions'].keys()):
-        if dist_exclude_pattern.match(dist):
-            logger.debug("excluded distribution %s", dist)
+
+class PackageDistReleaseData:
+    def __init__(self):
+        self.data = {}
+        self.releases = set()
+        self.distributions = set()
+        self.excluded_distributions = set()
+        self.excluded_packages = set()
+
+        self.dist_exclude_pattern = re.compile("win_cross|windows")
+        self.package_exclude_pattern = re.compile("php-ZendFramework2-.*|mhvtl.*|mingw.*|.*-dbg|.*-debug.*|.*-doc|.*-devel|.*-dev")
+
+
+    def generate(self, inputdata):
+        # input:
+        # '16.2':
+        #   'packages': {
+        #     package1: [ dist1, dist2, ... ],
+        #     package2: [ dist1, dist2, ... ],
+        #     ...
+        #   },
+        #   'distributions': {
+        #     distribution_name: [ package1, package2, ... ],
+        #     ...
+        #   },
+        # }
+
+        # output:
+        #   package1: {
+        #     dist1: [ "15.2", "16.2" ],
+        #     dist1: [ "16.2" ],
+        #     ...
+        #   },
+        #   package2: {
+        #     ...
+        #   },
+
+        for release in inputdata.keys():
+            self.add_release(release, inputdata[release])
+
+    def add_release(self, release, inputdata):
+        self.releases.add(release)
+        for package in inputdata['packages']:
+            for dist in inputdata['packages'][package]:
+                self.add(package, dist, release)
+
+    def add(self, package, dist, release):
+
+        if self.dist_exclude_pattern.match(dist):
+            self.excluded_distributions.add(dist)
+            return
+        if self.package_exclude_pattern.match(package):
+            self.excluded_packages.add(package)
+            return
+
+        self.distributions.add(dist)
+        if package not in self.data:
+            self.data[package] = {}
+        if dist not in self.data[package]:
+            self.data[package][dist] = []
+        self.data[package][dist].append(release)
+
+    def get_releases_string(self, package, dist):
+        if len(self.data[package][dist]) == 1:
+            return self.data[package][dist][0]
         else:
-            if dist_include.match(dist) and data['distributions'][dist]:
-                distributions.append(dist)
-            else:
-                logger.debug("excluded distribution %s (empty)", dist)
-    logger.debug(distributions)
+            minstr = str(min(self.data[package][dist]))
+            maxstr = str(max(self.data[package][dist]))
+            return minstr + "-" + maxstr
+
+    def is_package_for_dists(self, package, dists):
+        for dist in dists:
+            if dist in self.data[package]:
+                return True
+        return False
+
+    def get_packages_of_distributions(self, dists):
+        result = {}
+        for package in sorted(self.data.keys()):
+            if self.is_package_for_dists(package, dists):
+                result[package] = {}
+                for dist in dists:
+                    if dist in self.data[package]:
+                        result[package][dist] = self.get_releases_string(package, dist)
+        return result
+
+def get_matches(inputdata, refilter):
+    result = []
+    for i in inputdata:
+        if refilter.match(i):
+            result.append(i)
+    return result
+
+def generate_overview_table(packageDistReleaseData, dist_include):
     rows = {}
-    for package in sorted(data['packages'].keys()):
-        #if not package.endswith('-debuginfo'):
-        if package_exclude_pattern.match(package):
-            packages_excluded.append(package)
-        else:
-            package_dists = data['packages'][package]
-            #print "%-40s" % (package),
-            rows[package] = []
-            match= False
-            for dist in distributions:
-                if dist in package_dists:
-                    #print "X",
-                    rows[package].append('X')
-                    match=True
-                else:
-                    #print " ",
-                    rows[package].append(' ')
-            if not match:
-                del(rows[package])
-            #print
-    logger.debug("excluded packages: %s" % (packages_excluded))
+    distributions = sorted(get_matches(packageDistReleaseData.distributions, dist_include))
+    data = packageDistReleaseData.get_packages_of_distributions(distributions)
+    for package in sorted(data.keys()):
+        rows[package] = []
+        for dist in distributions:
+            if dist in data[package]:
+                rows[package].append(data[package][dist])
+            else:
+                rows[package].append(' ')
     latexTable = LatexTable(distributions, rows)
     return latexTable.get()
-
+
+def create_file(filename, data, filterregex):
+    with open(filename, "w") as file:
+        file.write(generate_overview_table(data, re.compile(filterregex)))
+        file.close()
+
 
 if __name__ == '__main__':
 
@@ -141,28 +204,28 @@ def generate_overview_table(data, dist_include):
 
     parser.add_argument('--debug', '-d', action='store_true', help="enable debugging output")
     parser.add_argument('--out', '-o', help="output directory", default=".")    
-    parser.add_argument('filename', help="JSON file containing package information")
+    parser.add_argument('filename', nargs="+", help="JSON files containing package information")
 
     args = parser.parse_args()
 
     if args.debug:
         logger.setLevel(logging.DEBUG)
-
-    with open(args.filename) as json_file:
-        json_data = json.load(json_file)
-    #print(json_data)
 
-    basename = os.path.basename(args.filename)
+    data = PackageDistReleaseData()
+
+    for filename in args.filename:
+        release = re.sub('.*bareos-([0-9.]+)-packages.json', r'\1', filename)
+        with open(filename) as json_file:
+            json_data = json.load(json_file)
+            data.add_release(release, json_data)
+
+    #print data.data
+
+    basename = 'bareos-packages'
     dest = args.out + '/' + basename
-    filename = dest.replace('.json','-table-redhat.tex')
-    with open(filename, "w") as file:
-        file.write(generate_overview_table(json_data, re.compile('CentOS.*|RHEL.*|Fedora.*')))
-        file.close()
-    filename = dest.replace('.json','-table-suse.tex')
-    with open(filename, "w") as file:
-        file.write(generate_overview_table(json_data, re.compile('openSUSE.*|SLE.*')))
-        file.close()
-    filename = dest.replace('.json','-table-debian.tex')
-    with open(filename, "w") as file:
-        file.write(generate_overview_table(json_data, re.compile('Debian.*|xUbuntu.*|Univention.*')))
-        file.close()
+    filenametemplate = dest + '-table-{0}.tex'
+    create_file(filenametemplate.format('redhat'), data, 'CentOS.*|RHEL.*')
+    create_file(filenametemplate.format('fedora'), data, 'Fedora_2.')
+    create_file(filenametemplate.format('suse'), data, 'openSUSE_13..|openSUSE_Leap_.*|SLE_10_SP4|SLE_11_SP4|SLE_12_SP1')
+    create_file(filenametemplate.format('debian'), data, 'Debian.*|Univention_4.*')
+    create_file(filenametemplate.format('ubuntu'), data, 'xUbuntu.*')