Add feature to generate diffs for helm values.yaml file between releases

This feature adds the capability to compare changes to the configuration options
in the helm values.yaml files between the current and previous release. This was
included in the 1.1 release but did not get merged. Here is the link to the
original PR: #3420

Fixes: #4384

Author: johnma14

scripts/tablegen.py

@@ -20,6 +20,7 @@
 import sys
 import os
 import re
+import requests
 
 from ruamel import yaml
 
@@ -35,6 +36,8 @@
 VALUES_YAML = "values.yaml"
 ISTIO_IO_DIR = os.path.abspath(__file__ + "/../../")
 CONFIG_INDEX_DIR = "content/docs/reference/config/installation-options/index.md"
+CONFIG_INDEX_DIFF_DIR = "content/docs/reference/config/installation-options-changes/index.md"
+CONFIG_IGNORE_LIST = ["global.hub"]
 
 def endOfTheList(context, lineNum, lastLineNum, totalNum):
     flag = 0
@@ -67,7 +70,15 @@ def endOfTheList(context, lineNum, lastLineNum, totalNum):
 
     return True, valueList
 
+# ordered dictionary to store the configuration options for the subcomponents of Istio. This
+# will be used to populate a new index.md
 prdict = collections.defaultdict(list)
+# ordered dictionary to store the differences of configuration options between the new 
+# index.md and the previous version (i.e, configurations options already listed in the index.md).
+od_diff = collections.defaultdict(list)
+od_diff_new = collections.defaultdict(list)
+od_diff_removed = collections.defaultdict(list)
+od_diff_unchanged = collections.defaultdict(list)
 
 def decode_helm_yaml(s):
     ret_val = ''
@@ -307,10 +318,243 @@ def sanitizeValueStr(value):
     if value != None and regex.search(value) != None:
         value = value.replace("|", "\|");
     return value
-        
-    
+
+# Compares the configuration option value from the newly discovered set of values (stored
+# in prdict dictionary) and its previous version (stored in index.md). If there is no
+# change in the configuration option value between the 2 versions, it will be ignored. If
+# there are any differences, we will store the differences (will track differences for key,
+# value and description of a configuration option) in the 'od_diff' dictionary. The values
+# stored in this dictionary will later be written to CONFIG_INDEX_DIFF_DIR.
+# 
+# The difference between the configuration option values is stored in the CONFIG_INDEX_DIFF_DIR
+# in the format:
+# |  KEY | OLD DEFAULT VALUE | NEW DEFAULT VALUE | OLD DESCRIPTION | NEW DESCRIPTION |
+# | ------ | ------------ | ------------ | ------------ | ------------ |
+# | Key | oldValue | newValue | oldDesc | newDesc |
+# 
+# If a configuration option is present only in the latest version, then the oldKey, oldValue
+# and oldDescription will be represented as 'n/a' (vice-versa applies to newKey, newValue and
+# newDescription). 
+#
+# oValue - configuration option from the existing index.md 
+# nValue - configuration option from the current processing of configuration options to be
+#          stored in a new version of index.md
+# k - istio component name for which these configuration options are being processed. This is
+#     used to populate the contents of 'od_diff' dictionary.
+#
+def compareValues(oValue, nValue, k):
+    # oValue and nVAlue contains configuration option in the format: 
+    # '| `<Key>` | `<Value>` | `<Description>` |
+    # This needs to be split in order to get the Key, Value and Description values to compare.
+    oldKey = ''
+    oldValue = ''
+    oldDesc = ''
+
+    newKey = ''
+    newValue = ''
+    newDesc = ''
+
+    key = None
+ 
+    if nValue is not None:   
+        groups = re.search("\| \`(.*)\` \| \`(.*)\` \| (.*) |", nValue.strip())
+        if groups:
+            newKey = groups.group(1)
+            newValue = groups.group(2)
+            newDesc = groups.group(3)
+  
+    if oValue is not None and nValue is not None:
+        if len(oValue) == 1:
+            item = oValue[0]
+
+            if item == nValue:
+                key = newKey
+                oValue.remove(item)
+                od_diff_unchanged[k].append("| `%s` | `%s` | %s |" % (newKey, newValue.rstrip(), newDesc))
+            else:
+                groups = re.search("\| \`(.*)\` \| \`(.*)\` \|\s*(.*)\s*\|", item.strip())
+                if groups:
+                    oldKey = groups.group(1)
+                    oldValue = groups.group(2)
+                    oldDesc = groups.group(3)
+                    key = oldKey
+
+                if oldKey in CONFIG_IGNORE_LIST:
+                    oValue.remove(item) 
+                    return key
+
+                if oldValue != newValue:
+                    if oldValue is None:
+                        oldValue = 'n/a'
+                    if newValue is None:
+                        newValue = 'n/a'
+
+                    if oldDesc.strip() != newDesc.strip():
+                        if (newDesc == None or newDesc == '') and (oldDesc is None or oldDesc == ''):
+                            pass
+                        if oldDesc is None:
+                            oldDesc = 'n/a'
+                        if newDesc is None or newDesc == '':
+                            newDesc = 'n/a'
+                    oValue.remove(item)
+                    od_diff[k].append("| `%s` | `%s` | `%s` | %s | %s |" % (newKey, oldValue.rstrip(), newValue.rstrip(), oldDesc, newDesc))
+                else:
+		    # This is the case where values are the same but descriptions are different. Right now, there is nothing more to do since
+		    # we do not care about displaying values that haven't changed between releases.
+                    oValue.remove(item)
+               
+                    #od_diff_unchanged[k].append("| `%s` | `%s` | %s |" % (newKey, newValue.rstrip(), newDesc))   
+        else:
+            foundItem = 'false'
+            for item in oValue:
+                if item == nValue:
+                    key = newKey
+                    oValue.remove(item)
+                    od_diff_unchanged[k].append("| `%s` | `%s` | %s |" % (newKey, newValue.rstrip(), newDesc))
+                    foundItem = 'true'
+                    break
+                else:
+                    groups = re.search("\| \`(.*)\` \| \`(.*)\` \|\s*(.*)\s*\|", item.strip())
+                    if groups:
+                        oldKey = groups.group(1)
+                        oldValue = groups.group(2)
+                        oldDesc = groups.group(3)
+
+                    if oldKey == newKey:
+                        if oldValue == newValue and oldDesc != newDesc:
+                            key = newKey
+                            od_diff[k].append("| `%s` | `%s` | `%s` | %s | %s |" % (newKey, oldValue.rstrip(), newValue.rstrip(), oldDesc, newDesc))
+                            oValue.remove(item)
+                            foundItem = 'true'
+                            break
+
+            if foundItem == 'false':
+                od_diff_new[k].append("| `%s` | `%s` | %s |" % (newKey, newValue.rstrip(), newDesc))
+    elif oValue is None:
+        key = newKey
+        od_diff_new[k].append("| `%s` | `%s` | %s |" % (newKey, newValue.rstrip(), newDesc))   
+    elif nValue is None:
+        for item in oValue:
+            groups = re.search("\| \`(.*)\` \| \`(.*)\` \|\s*(.*)\s*\|", item.strip())
+            if groups:
+                oldKey = groups.group(1)
+                oldValue = groups.group(2)
+                oldDesc = groups.group(3)
+
+            key = oldKey
+            od_diff_removed[k].append("| `%s` | `%s` | %s |" % (oldKey, oldValue.rstrip(), oldDesc))
+
+    return key
+
+# 
+# Get the previous release number so that we can retrieve the index.md for that
+# release. The release branches are tagged in the following format: release-<number>
+#
+def getPreviousRelease():
+    req = requests.get('https://api.github.com/repos/istio/istio.io/branches')
+    jsonData = req.json()
+    previousRelease = 0.0
+
+    for x in jsonData:
+        releaseName = x['name']
+        if releaseName.startswith('release-'):
+            releaseNum = releaseName.split('release-')
+            if releaseNum[1] > previousRelease:
+                previousRelease = releaseNum[1]
+    return previousRelease
+
+#
+# Get the index.md for the previous release.
+#
+def getContentFromPreviousRelease(releaseName):
+    istio_url = 'https://raw.githubusercontent.com/istio/istio.io/release-' + releaseName +'/content/docs/reference/config/installation-options/index.md'
+    req = requests.get(istio_url)
+    content = req.text
+    indexMap = collections.defaultdict(list)
+
+    # store all the configurations options from the index.md file into the indexMap
+    # dictionary. This will be used to compare the values with the latest version
+    # of configuration options.
+    data = content.split('\n')
+    for d in data:
+        if d.rstrip() != '' and d != '| Key | Default Value | Description |' and d != '| --- | --- | --- |' and d[0:1] == '|' and d[-1] == '|':
+            groups = re.search("\| \`(.*)\` \| \`(.*)\` \| (.*) |", d.strip())
+            if groups:
+                key = groups.group(1)
+                if key in indexMap:
+                    value = indexMap.get(key)
+                    value.append(d.strip())
+                else:
+                    indexMap[key].append(d.strip())
+    return indexMap
+
+def writeVersionDiffs(index_diff_file):
+    meta = ""
+
+    for d in index_diff_file:
+        meta = meta + d
+        if "<!-- AUTO-GENERATED-START -->" in d:
+            break
+
+    index_diff_file.seek(0)
+    index_diff_file.write(meta)
+
+    '''
+    if od_diff_unchanged:
+        index_diff_file.write('\n## Unmodified configuration options\n')
+
+    for k, v in od_diff_unchanged.items():
+        index_diff_file.write("\n### Unmodified `%s` key/value pairs\n\n" % k)
+        index_diff_file.write('| Key | Default Value | Description |\n')
+        index_diff_file.write('| --- | --- | --- |\n')
+
+        for value in v:
+            index_diff_file.write('%s\n' % (value))
+    '''
+
+    if od_diff:
+        index_diff_file.write('\n## Modified configuration options\n')
+
+    for k, v in od_diff.items():
+        index_diff_file.write("\n### Modified `%s` key/value pairs\n\n" % k) 
+        index_diff_file.write('| Key | Old Default Value | New Default Value | Old Description | New Description |\n')
+        index_diff_file.write('| --- | --- | --- | --- | --- |\n')
+
+        for value in v:
+            index_diff_file.write('%s\n' % (value))
+
+    if od_diff_new:
+        index_diff_file.write('\n## New configuration options\n')
+
+    for k, v in od_diff_new.items():
+        index_diff_file.write("\n### New `%s` key/value pairs\n\n" % k)
+        index_diff_file.write('| Key | Default Value | Description |\n')
+        index_diff_file.write('| --- | --- | --- |\n')
+
+        for value in v:
+            index_diff_file.write('%s\n' % (value))
+
+    if od_diff_removed:
+        index_diff_file.write('\n## Removed configuration options\n')
+
+    for k, v in od_diff_removed.items():
+        index_diff_file.write("\n### Removed `%s` key/value pairs\n\n" % k)
+        index_diff_file.write('| Key | Default Value | Description |\n')
+        index_diff_file.write('| --- | --- | --- |\n')
+
+        for value in v:
+            index_diff_file.write('%s\n' % (value))
+
+    index_diff_file.write("\n<!-- AUTO-GENERATED-END -->\n")
+
 with open(os.path.join(ISTIO_IO_DIR, CONFIG_INDEX_DIR), 'r') as f:
     endReached = False
+    key = ''
+    # A list used to track the configuration options that has been compared and processed when going
+    # through the configurations processed in the latest version 
+    indexList = []
+    previousRelease = getPreviousRelease()
+    indexMap = getContentFromPreviousRelease(previousRelease)
 
     data = f.read().split('\n')
     for d in data:
@@ -332,11 +576,34 @@ def sanitizeValueStr(value):
         print '| Key | Default Value | Description |'
         print '| --- | --- | --- |'
         for value in v:
-            print ('%s' % (value))
+            print ('%s' % (value))  
+            # Compare configuration option values from the latest version
+            # with the older version.
+            groups = re.search("\| \`(.*)\` \| \`(.*)\` \| (.*) |", value.strip())
+            if groups:
+                key = groups.group(1)
+                indexValue = indexMap.get(key)
+               
+                indexList.append(compareValues(indexValue, value, k))
         print ('')
 
     for d in data:
         if "<!-- AUTO-GENERATED-END -->" in d:
             endReached = True
         if endReached:
             print d
+
+    # We want to include any configuration options that was discovered in
+    # the older version but not available in the current version
+    for k in indexMap.keys():
+        key = k.split('.')[0]
+        indexList.append(compareValues(indexMap.get(k), None, key))   
+    
+    # This index.md file is used to track the differences of configuration
+    # option values between the current and previous release. All the
+    # differences in configuration option values between the current
+    # and previous release (tracked in the 'od_diff' dictionary) will be
+    # written to the index.md file
+    index_diff_file = open(os.path.join(ISTIO_IO_DIR, CONFIG_INDEX_DIFF_DIR), 'r+')
+    writeVersionDiffs(index_diff_file)
+    index_diff_file.close()