Maximise compatibility in the tool (#20)

* Write a compatibility test

* more parser stuff

* Make it ignore small parser errors

* Move how this works

* add support for a HTTP mirror I wrote for testing

* Fixes for this + some fixes to my HTTP mirror

* check if dev env

* doesn't need to be set

* there we goooo, this is more compatible

* more stuff

280/282 on the compatibility test boiiiii

* Further compatibility things

* change where it is commited to

* test the do-vue branch I just made

* Change to master

* Update package-lock.json

* fix some bugs

* Fix some big issues with rendering

* oops

* Fixes a bug with printf

* Fix the bug that made Jenkins fail

* make message clearer

* new line here

* Make this clearer

* just this one env exposed

* Update src/kubernetes-tool/utils/helm_parts/document_parser.ts

Co-Authored-By: Matt (IPv4) Cowley <me@mattcowley.co.uk>

* bug fixes

* can now been inline

package.json

@@ -43,7 +43,6 @@
     "parcel-bundler": "1.12.3",
     "printj": "^1.2.2",
     "prismjs": "^1.17.1",
-    "semver": "^6.3.0",
     "vue": "^2.6.10",
     "vue-hot-reload-api": "^2.3.3",
     "vue-prism-component": "^1.1.1",

src/kubernetes-tool/mount.js

@@ -17,6 +17,7 @@ limitations under the License.
 import "babel-polyfill"
 import "abortcontroller-polyfill/dist/polyfill-patch-fetch"
 
+import "./utils/compatibility_test"
 import Vue from "vue"
 import App from "./templates/app.vue"
 import i18n from "./i18n"

src/kubernetes-tool/utils/compatibility_test.ts

@@ -0,0 +1,28 @@
+import { fs } from "../utils/helm_parts/utils"
+// @ts-ignore
+import { HelmCoreParser } from "./helm"
+
+// Runs the compatibility test.
+async function test() {
+    let errors = ""
+    let passed = ""
+    let total = 0
+    let works = 0
+    const promises = []
+    for (const item of await fs.ls("stable")) {
+        if (!item.file) {
+            total++
+            promises.push(new HelmCoreParser({}, item.path).promise.then(() => {
+                passed += item.path + "\n"
+                works++
+            }).catch(err => {
+                errors += `${item.path}: ${err}\n`
+            }))
+        }
+    }
+    await Promise.all(promises)
+    console.log(`${works}/${total}\n---\nFailed\n---\n${errors === "" ?  "No errors found!\n" : errors}\nPassed\n---\n${passed}`)
+}
+
+// @ts-ignore
+window.compatibiltiyTest = test

src/kubernetes-tool/utils/gitHttpMirrorFs.ts

@@ -0,0 +1,63 @@
+/*
+Copyright 2019 DigitalOcean
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Defines a very basic filesystem using git-http-mirror.
+export default class GitHTTPMirrorFS {
+    public alias: string
+    public hostname: string
+
+    // Constructs the class.
+    public constructor(alias: string, hostname: string) {
+        this.alias = alias
+        this.hostname = hostname
+    }
+
+    // Lists the folder specified.
+    public async ls(folder: string): Promise<{
+        file: boolean;
+        path: string;
+        name: string;
+    }[]> {
+        const items: {
+            file: boolean;
+            path: string;
+            name: string;
+        }[] = []
+        const res = await fetch(`${this.hostname}/${this.alias}/${folder}`, {headers: {
+            Accept: "application/json",
+        }})
+        if (res.headers.get("is-dir-listing") === "false") {
+            return []
+        }
+        const json = await res.json()
+        for (const item of json) {
+            const itemResult = await fetch(`${this.hostname}/${this.alias}/${folder}/${item}`)
+            items.push({
+                file: itemResult.headers.get("is-dir-listing") !== "true",
+                name: item,
+                path: `${folder}/${item}`,
+            })
+        }
+        return items
+    }
+
+    // Gets the item specified. Returns null if it is not found.
+    public async get(fp: string): Promise<string | undefined> {
+        const res = await fetch(`${this.hostname}/${this.alias}/${fp}`)
+        if (!res.ok) return undefined
+        return await res.text()
+    }
+}

src/kubernetes-tool/utils/helm_parts/core_parser.ts

@@ -22,14 +22,6 @@ import helmCache from "./helm_cache"
 import asyncLock from "async-lock"
 const lock = new asyncLock()
 
-// Patches for values.
-const valuesPatches = {
-    "stable/rethinkdb": (values: any) => {
-        values.cluster.readinessProbe = ""
-        return values
-    },
-} as Record<string, (values: any) => any>
-
 // Defines the Helm core parser.
 export default class HelmCoreParser {
     public context: HelmDocumentParser
@@ -69,7 +61,11 @@ export default class HelmCoreParser {
 
         // Loads the values.yaml.
         const valuesYaml = safeLoad(unparsedValuesYaml) as Record<string, any>
-        this.context.context.Values = valuesPatches[path] ? valuesPatches[path](valuesYaml) : valuesYaml
+        this.context.context.Values = valuesYaml
+
+        // Fixes for potential bufs.
+        valuesYaml.cluster = valuesYaml.cluster ? {...valuesYaml.cluster, readinessProbe: ""} : {readinessProbe: ""}
+        valuesYaml.global = true
 
         // Sets the release context.
         this.context.context.Release = {

src/kubernetes-tool/utils/helm_parts/document_parser.ts

@@ -35,15 +35,40 @@ export default class DocumentParser {
     // Handles each specific token.
     private _handleToken(token: Token, additionalQuotes: Quote[]): string {
         // Some initialisation to get the function and arguments.
-        const tokens: Token[] = []
         let data = token.data.trim()
+        const args: (string | Quote)[] = []
+
+        // Handles brackets in the tool.
+        let dSplit: (string | Quote)[] = [data]
         for (;;) {
-            const m = data.match(/\((.+?)\)/)
-            if (!m) break
-            tokens.push(new Token(m[1]))
-            data = data.replace(m[0], "__TOKEN")
+            const results: boolean[] = []
+            const newdSplit: (string | Quote)[] = []
+            for (const d of dSplit) {
+                if (typeof d !== "string") {
+                    results.push(true)
+                    newdSplit.push(d)
+                    continue
+                }
+                const m = d.match(/\((.+?)\)/)
+                if (!m) {
+                    results.push(true)
+                    newdSplit.push(d)
+                    continue
+                }
+                const remainingData = d.split(m[0])
+                const middle = new Quote(this._handleToken(new Token(m[1]), []))
+                newdSplit.push(remainingData[0], middle, remainingData[1])
+                results.push(false)
+            }
+            dSplit = newdSplit
+            if (results.every(x => x)) break
+        }
+
+        // Splits the data properly.
+        for (const d of dSplit) {
+            if (typeof d === "string") args.push(...d.split(" "))
+            else args.push(d)
         }
-        const args: (string | Quote)[] = data.split(" ")
 
         // Handles quotes.
         let quoteParts: {
@@ -56,9 +81,7 @@ export default class DocumentParser {
             toAdd: Quote;
         }[] = []
         for (const a in args) {
-            if (args[a] === "__TOKEN") {
-                args[a] = new Quote(this._handleToken(tokens.shift()!, []))
-            } else if (typeof args[a] === "string") {
+            if (typeof args[a] === "string") {
                 const strArg = args[a] as string
                 if (strArg.startsWith("\"")) {
                     quoteParts.push({
@@ -84,12 +107,17 @@ export default class DocumentParser {
         for (const q of additionalQuotes) args.push(q)
 
         // Gets the function.
-        const func: string = args.shift()! as string
+        let func = args.shift()! as string
+        if (((func as unknown) as Quote).text) {
+            func = ((func as unknown) as Quote).text
+        }
 
         // Runs the function.
         if (functions[func] === undefined) {
             if (func.startsWith(".")) return String(this.helmdef2object(func))
-            throw new Error(`${func} - Unknown command!`)
+
+            // We should return here because even though this may not be fully accurate, it allows for an as accurate as possible result.
+            return ""
         }
         const exec = functions[func](this, args, token)
         return exec

src/kubernetes-tool/utils/helm_parts/functions/define.ts

@@ -20,6 +20,15 @@ import { Token } from "../tokeniser"
 
 export default (parser: DocumentParser, args: (string | Quote)[], token: Token): string => {
     const full = parser.processArg(args[0])
-    parser.templateContext[full] = parser.handleTokens(token.inner!).trim()
+    const numbers: string[] = []
+    for (const ti in token.inner!) {
+        const t = token.inner![ti]
+        if (typeof t === "string" && t.trim().startsWith("{{/*")) numbers.push(ti)
+    }
+    const newTokens: (string | Token)[] = []
+    for (const ti in token.inner!) {
+        if (!numbers.includes(ti)) newTokens.push(token.inner![Number(ti)])
+    }
+    parser.templateContext[full] = parser.handleTokens(newTokens).trim()
     return ""
 }

src/kubernetes-tool/utils/helm_parts/functions/if.ts

@@ -17,7 +17,6 @@ limitations under the License.
 import DocumentParser from "../document_parser"
 import { Quote, OperatorManager } from "../utils"
 import { Token } from "../tokeniser"
-import * as semver from "semver"
 
 // Processes a condition. Is it true?
 const processCondition = (parser: DocumentParser, args: (string | Quote)[]): boolean => {
@@ -36,9 +35,6 @@ const processCondition = (parser: DocumentParser, args: (string | Quote)[]): boo
     // Does one string contain another?
     let contain = true
 
-    // Is this a semver check?
-    let semverFlag = false
-
     // Is this a empty check?
     let empty = false
 
@@ -80,8 +76,7 @@ const processCondition = (parser: DocumentParser, args: (string | Quote)[]): boo
             break
         }
         case "semverCompare": {
-            semverFlag = true
-            break
+            return false
         }
         case "empty": {
             empty = true
@@ -96,7 +91,8 @@ const processCondition = (parser: DocumentParser, args: (string | Quote)[]): boo
                 // This *should* be the condition.
                 return Boolean(parser.helmdef2object(operator))
             } else {
-                throw new Error(`"${operator}" - Invalid operator!`)
+                // Returns false if it doesn't understand.
+                return false
             }
         }
     }
@@ -110,9 +106,6 @@ const processCondition = (parser: DocumentParser, args: (string | Quote)[]): boo
     // Handles include.
     if (include) return dataParts[0]
 
-    // Handles semver.
-    if (semverFlag) return semver.eq(dataParts[0], dataParts[1])
-
     // If this is a not statement, we only need to worry about the first arg.
     if (not) return !Boolean(dataParts[0])
 

src/kubernetes-tool/utils/helm_parts/functions/include.ts

@@ -17,4 +17,4 @@ limitations under the License.
 import DocumentParser from "../document_parser"
 import { Quote } from "../utils"
 
-export default (parser: DocumentParser, args: (string | Quote)[]): string => String(parser.processArg(args[0]))
+export default (parser: DocumentParser, args: (string | Quote)[]): string => String(typeof args[0] === "string" ? parser.templateContext[args[0]] : parser.templateContext[args[0].text])

src/kubernetes-tool/utils/helm_parts/functions/indent.ts

@@ -19,7 +19,7 @@ import { Quote } from "../utils"
 
 export default (parser: DocumentParser, args: (string | Quote)[]): string => {
     const toRepeat = " ".repeat(Number(parser.processArg(args[0])))
-    const dataSplit = parser.processArg(args[1]).split("\n")
+    const dataSplit = String(parser.processArg(args[1])).split("\n")
     for (const part in dataSplit) {
         dataSplit[part] = `${toRepeat}${dataSplit[part]}`
     }

src/kubernetes-tool/utils/helm_parts/functions/index.ts

@@ -34,8 +34,10 @@ import printf from "./printf"
 import include from "./include"
 import replace from "./replace"
 import b64enc from "./b64enc"
+import { randAlphaNum, randNum, randAlpha } from "./randAlphaNum"
 
 export default {
     env, uuidv4, trimSuffix, if: if_, range, default: default_, quote, define,
-    template, trunc, indent, toYaml, printf, include, replace, b64enc,
+    template, trunc, indent, toYaml, printf, include, replace, b64enc, randAlphaNum,
+    randNum, randAlpha,
 } as Record<string, (parser: DocumentParser, args: (string | Quote)[], token: Token) => string>

src/kubernetes-tool/utils/helm_parts/functions/printf.ts

@@ -22,5 +22,11 @@ export default (parser: DocumentParser, args: (string | Quote)[]): string => {
     const formatter = parser.processArg(args.shift()!)
     const transformedArgs: any[] = []
     for (const a of args) transformedArgs.push(parser.processArg(a))
-    return printj.sprintf(formatter, ...transformedArgs)
+    try {
+        // You can't inline try and catch on JS, I tried (no pun intended!)
+        return printj.sprintf(formatter, ...transformedArgs)
+    } catch (_) {
+        // Something is wrong with the formatter, this is not our issue.
+        return ""
+    }
 }

src/kubernetes-tool/utils/helm_parts/functions/randAlphaNum.ts

@@ -0,0 +1,19 @@
+/*
+Copyright 2019 DigitalOcean
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+export const randAlphaNum = (): string => "<random char (alphabet/number)>"
+export const randAlpha = (): string => "<random char (alphabet)>"
+export const randNum = (): string => "<random char (number)>"

src/kubernetes-tool/utils/helm_parts/tokeniser.ts

@@ -61,8 +61,8 @@ export class Tokeniser {
 
     // Finds the end statement.
     private _manageEnd(matches: RegExpMatchArray[]): RegExpMatchArray[] | undefined {
-        // Tells the parser to skip the next end.
-        let skip = false
+        // Tells the parser the level.
+        let level = 1
 
         // Contains either 1 end, or else's and a end.
         const returned: RegExpMatchArray[] = []
@@ -73,16 +73,15 @@ export class Tokeniser {
             if (!m) break
             const t = m[1].split(/ +/g)[0].toLowerCase()
             if (requireEnd.includes(t)) {
-                skip = true
+                level++
             } else if (t === "else") {
-                if (!skip) returned.push(m)
+                if (level === 1) returned.push(m)
             } else if (t === "end") {
-                if (skip) {
-                    skip = false
-                } else {
+                if (level === 1) {
                     returned.push(m)
                     return returned
-                } 
+                }
+                level--
             }
         }
     }

src/kubernetes-tool/utils/helm_parts/utils.ts

@@ -14,8 +14,12 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+// Defines the mirror hostname.
+let mirrorHostname = process.env.NODE_ENV === "development" ? "http://localhost:8001" : null
+
 // Imports needed stuff.
 import GitHubFS from "../githubFs"
+import GitHTTPMirrorFS from "../gitHttpMirrorFs"
 
 // The operator manager. Allows for operations to safely be evaled between 2 objects.
 export class OperatorManager {
@@ -43,7 +47,7 @@ export class Quote {
 }
 
 // A statement in Helm.
-export const helmStatement = /{{[ -]*([^}]+)[ -]*}}/g
+export const helmStatement = /{{[ -]*((?:[^}]|\n)+)[ -]*}}/g
 
 // Defines the filesystem for the Helm Charts official repository.
-export const fs = new GitHubFS("helm/charts")
+export const fs = mirrorHostname ? new GitHTTPMirrorFS("helm", mirrorHostname) : new GitHubFS("helm/charts")