Tablefy Output and Add Demo CI Workflow

.github/workflows/demo.yml

@@ -0,0 +1,63 @@
+name: Demo on openconfig/public
+
+on:
+  pull_request:
+    branches: [ main ]
+
+jobs:
+
+  pattern:
+    name: pattern statement
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+
+    steps:
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Setup Python
+      uses: actions/setup-python@v2
+      with:
+          python-version: '3.x'
+
+    - name: Set up pyang
+      run: pip3 install pyang
+
+    - name: Get public repo
+      run: git clone https://github.com/openconfig/public.git ~/tmp/public
+
+    - name: Demo output on openconfig/public
+      continue-on-error: true
+      run: |
+          OCDIR=~/tmp/public pytests/pattern_test.sh
+
+  posix-pattern:
+    name: posix-pattern statement
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+
+    steps:
+    - name: Set up Go 1.x
+      uses: actions/setup-go@v2
+      # Use default go version so that we don't have to update it every time a new one comes out.
+      id: go
+
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v2
+
+    - name: Get dependencies
+      run: |
+        go get -v -t -d ./...
+
+    - name: Build
+      run: go build -v ./...
+
+    - name: Get public repo
+      run: git clone https://github.com/openconfig/public.git ~/tmp/public
+
+    - name: Demo output on openconfig/public
+      continue-on-error: true
+      run: |
+          go run gotests/main.go -model-root ~/tmp/public testdata/regexp-test.yang

README.md

@@ -1,12 +1,26 @@
 # Pattern Statement Tests for OpenConfig YANG models
 
-**Under implementation.**
-
 Tests [pattern statement](https://tools.ietf.org/html/rfc7950#section-9.4.5)
 using a [pyang](https://github.com/mbj4668/pyang) plugin and
 [oc-ext:posix-pattern](https://github.com/openconfig/public/blob/master/release/models/openconfig-extensions.yang#L114)
 using [goyang](https://github.com/openconfig/goyang).
 
+## Demo CI Workflow
+
+There is a demo CI workflow that runs on Pull Requests. They are used to demo
+the result of running the tests on the current YANG models. If they cause an
+expected test failure (perhaps because you added an uncaught corner case), it
+will not block merge and a minor version increment will be given.
+
+## Releases
+
+Releases are synchronized with the current OpenConfig YANG models. If new
+breaking tests are added (e.g. test cases handled incorrectly by a current
+pattern regex), then the minor version must be incremented.
+
+At this time, major version updates are not anticipated, but could occur as a
+result of major changes to the repository.
+
 --------------------------------------------------------------------------------
 
 [OpenConfig YANG models](https://github.com/openconfig/public/blob/master/README.md)

go.mod

@@ -4,7 +4,8 @@ go 1.15
 
 require (
 	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
-	github.com/openconfig/gnmi v0.0.0-20201217212801-57b8e7af2d36
+	github.com/google/go-cmp v0.5.0
+	github.com/openconfig/gnmi v0.0.0-20201217212801-57b8e7af2d36 // indirect
 	github.com/openconfig/goyang v0.2.4
 	github.com/openconfig/ygot v0.9.0
 )

gotests/main.go

@@ -19,10 +19,11 @@ package main
 
 import (
 	"flag"
+	"fmt"
+	"os"
 
 	log "github.com/golang/glog"
 	"github.com/openconfig/pattern-regex-tests/gotests/patterncheck"
-	"github.com/openconfig/ygot/util"
 )
 
 var (
@@ -32,17 +33,25 @@ var (
 func main() {
 	flag.Parse()
 
+	code := 0
+	defer func() {
+		os.Exit(code)
+	}()
+
 	if *modelRoot == "" {
 		log.Error("Must supply model-root path")
 	}
 
-	if err := patterncheck.CheckRegexps(flag.Args(), []string{*modelRoot}); err != nil {
-		if errors, ok := err.(util.Errors); ok {
-			for _, err := range errors {
-				log.Errorln(err)
-			}
-		} else {
-			log.Exit(err)
+	failureMessages, err := patterncheck.CheckRegexps(flag.Args(), []string{*modelRoot})
+	if err != nil {
+		log.Exit(err)
+	}
+	if len(failureMessages) != 0 {
+		code = 1
+		fmt.Fprintln(os.Stderr, "| leaf | typedef | error |")
+		fmt.Fprintln(os.Stderr, "| --- | --- | --- |")
+		for _, msg := range failureMessages {
+			fmt.Fprintln(os.Stderr, msg)
 		}
 	}
 }

gotests/patterncheck/patterncheck.go

@@ -26,8 +26,8 @@ import (
 )
 
 var (
-	passCaseExt = regexp.MustCompile(`\w:pattern-test-pass`)
-	failCaseExt = regexp.MustCompile(`\w:pattern-test-fail`)
+	passCaseExt = regexp.MustCompile(`\w+:pattern-test-pass`)
+	failCaseExt = regexp.MustCompile(`\w+:pattern-test-fail`)
 )
 
 // YANGLeaf is a structure used to describe a particular leaf of YANG schema.
@@ -45,39 +45,42 @@ type RegexpTest struct {
 // CheckRegexps tests mock input data against a set of leaves that have pattern
 // test cases specified for them. It ensures that the regexp compiles as a
 // POSIX regular expression according to the OpenConfig style guide.
-func CheckRegexps(yangfiles, paths []string) error {
+func CheckRegexps(yangfiles, paths []string) ([]string, error) {
 	yangE, errs := yangutil.ProcessModules(yangfiles, paths)
 	if len(errs) != 0 {
-		return fmt.Errorf("could not parse modules: %v", errs)
+		return nil, fmt.Errorf("could not parse modules: %v", errs)
 	}
 	if len(yangE) == 0 {
-		return fmt.Errorf("did not parse any modules")
+		return nil, fmt.Errorf("did not parse any modules")
 	}
 
-	var errs2 util.Errors
+	var patternErrs util.Errors
+	var allFailMessages []string
 	for _, mod := range yangE {
 		for _, entry := range mod.Dir {
-			if err := checkEntryPatterns(entry); err != nil {
-				errs2 = util.AppendErr(errs2, err)
+			if failMessages, err := checkEntryPatterns(entry); err != nil {
+				patternErrs = util.AppendErr(patternErrs, err)
+			} else {
+				allFailMessages = append(allFailMessages, failMessages...)
 			}
 		}
 	}
-	if len(errs2) == 0 {
-		return nil
+	if len(patternErrs) != 0 {
+		return nil, patternErrs
 	}
-	return errs2
+	return allFailMessages, nil
 }
 
-func checkEntryPatterns(entry *yang.Entry) error {
+func checkEntryPatterns(entry *yang.Entry) ([]string, error) {
 	if entry.Kind != yang.LeafEntry {
-		return nil
+		return nil, nil
 	}
 
 	if len(entry.Errors) != 0 {
-		return fmt.Errorf("entry had associated errors: %v", entry.Errors)
+		return nil, fmt.Errorf("entry had associated errors: %v", entry.Errors)
 	}
 
-	var errs util.Errors
+	var failMessages []string
 	for _, ext := range entry.Exts {
 		var wantMatch bool
 		switch {
@@ -93,7 +96,7 @@ func checkEntryPatterns(entry *yang.Entry) error {
 		if len(entry.Type.Type) == 0 {
 			var err error
 			if gotMatch, err = checkPatterns(ext.Argument, entry.Type.POSIXPattern); err != nil {
-				return err
+				return nil, err
 			}
 		} else {
 			// Handle unions.
@@ -107,26 +110,23 @@ func checkEntryPatterns(entry *yang.Entry) error {
 				}
 				matches, err := checkPatterns(ext.Argument, membertype.POSIXPattern)
 				if err != nil {
-					return err
+					return nil, err
 				}
 				gotMatch = gotMatch || matches
 			}
 		}
 
-		matchDesc := fmt.Sprintf("%q doesn't match type %s (leaf %s)", ext.Argument, entry.Type.Name, entry.Name)
-		if gotMatch {
-			matchDesc = fmt.Sprintf("%q matches type %s (leaf %s)", ext.Argument, entry.Type.Name, entry.Name)
+		matchDesc := fmt.Sprintf("| `%s` | `%s` | `%s` matched but shouldn't |", entry.Name, entry.Type.Name, ext.Argument)
+		if !gotMatch {
+			matchDesc = fmt.Sprintf("| `%s` | `%s` | `%s` did not match |", entry.Name, entry.Type.Name, ext.Argument)
 		}
 
 		if gotMatch != wantMatch {
-			errs = util.AppendErr(errs, fmt.Errorf("fail: %s", matchDesc))
+			failMessages = append(failMessages, matchDesc)
 		}
 		log.Infof("pass: %s", matchDesc)
 	}
-	if len(errs) == 0 {
-		return nil
-	}
-	return errs
+	return failMessages, nil
 }
 
 // checkPatterns compiles all given POSIX patterns, and returns true if

gotests/patterncheck/patterncheck_test.go

@@ -17,40 +17,52 @@ package patterncheck
 import (
 	"testing"
 
-	"github.com/openconfig/gnmi/errdiff"
+	"github.com/google/go-cmp/cmp"
 )
 
 func TestCheckRegexps(t *testing.T) {
 	tests := []struct {
 		desc             string
 		inFiles          []string
 		inPaths          []string
-		wantErrSubstring string
+		wantFailMessages []string
 	}{{
 		desc:    "passing cases",
 		inFiles: []string{"testdata/passing.yang"},
 		inPaths: []string{"../../testdata"},
 	}, {
-		desc:             "simple leaf fail",
-		inFiles:          []string{"testdata/simple-leaf-fail.yang"},
-		inPaths:          []string{"../../testdata"},
-		wantErrSubstring: `"ipv4" matches type string (leaf ipv-0), fail: "ipv6" doesn't match type string (leaf ipv-0)`,
+		desc:    "simple leaf fail",
+		inFiles: []string{"testdata/simple-leaf-fail.yang"},
+		inPaths: []string{"../../testdata"},
+		wantFailMessages: []string{
+			"| `ipv-0` | `string` | `ipv4` matched but shouldn't |",
+			"| `ipv-0` | `string` | `ipv6` did not match |",
+		},
 	}, {
-		desc:             "union leaf fail",
-		inFiles:          []string{"testdata/union-leaf-fail.yang"},
-		inPaths:          []string{"../../testdata"},
-		wantErrSubstring: `fail: "ipv4" matches type ip-string-typedef (leaf ipv-0), fail: "ipv5" doesn't match type ip-string-typedef (leaf ipv-0)`,
+		desc:    "union leaf fail",
+		inFiles: []string{"testdata/union-leaf-fail.yang"},
+		inPaths: []string{"../../testdata"},
+		wantFailMessages: []string{
+			"| `ipv-0` | `ip-string-typedef` | `ipv4` matched but shouldn't |",
+			"| `ipv-0` | `ip-string-typedef` | `ipv5` did not match |",
+		},
 	}, {
-		desc:             "derived string type fail",
-		inFiles:          []string{"testdata/derived-string-fail.yang"},
-		inPaths:          []string{"../../testdata"},
-		wantErrSubstring: `fail: "ipV4" doesn't match type ipv4-address-str (leaf ipv-0), fail: "ipV4-address" matches type ipv4-address-str (leaf ipv-0)`,
+		desc:    "derived string type fail",
+		inFiles: []string{"testdata/derived-string-fail.yang"},
+		inPaths: []string{"../../testdata"},
+		wantFailMessages: []string{
+			"| `ipv-0` | `ipv4-address-str` | `ipV4` did not match |",
+			"| `ipv-0` | `ipv4-address-str` | `ipV4-address` matched but shouldn't |",
+		},
 	}}
 
 	for _, tt := range tests {
 		t.Run(tt.desc, func(t *testing.T) {
-			got := CheckRegexps(tt.inFiles, tt.inPaths)
-			if diff := errdiff.Substring(got, tt.wantErrSubstring); diff != "" {
+			got, err := CheckRegexps(tt.inFiles, tt.inPaths)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if diff := cmp.Diff(got, tt.wantFailMessages); diff != "" {
 				t.Errorf("(-got, +want):\n%s", diff)
 			}
 		})

pytests/pattern_test.sh

@@ -9,4 +9,13 @@ fi
 TEST_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 REPO_DIR="$TEST_DIR/.."
 
-pyang -p $OCDIR -p "$REPO_DIR/testdata" --msg-template="line {line}: {msg}" --plugindir "$REPO_DIR/pytests/plugins" --check-patterns "$REPO_DIR/testdata/regexp-test.yang"
+tmpstderr=$(mktemp)
+pyang -p $OCDIR -p "$REPO_DIR/testdata" --msg-template="| {line} | {msg} |" --plugindir "$REPO_DIR/pytests/plugins" --check-patterns "$REPO_DIR/testdata/regexp-test.yang" 2> $tmpstderr
+retcode=$?
+if [ $retcode -ne 0 ]; then
+  >&2 echo "| Line # | typedef | error |"
+  >&2 echo "| --- | --- | --- |"
+fi
+>&2 cat $tmpstderr
+rm $tmpstderr
+exit $retcode

pytests/plugins/pattern_test.py

@@ -57,18 +57,18 @@ def setup_ctx(self, ctx):
         # Test case failure states.
         error.add_error_code(
             'VALID_PATTERN_DOES_NOT_MATCH', ErrorLevel.MAJOR,
-            'type "%s" rejected valid pattern: "%s"')
+            '`%s` | `%s` did not match')
         error.add_error_code(
             'INVALID_PATTERN_MATCH', ErrorLevel.MAJOR,
-            'type "%s" accepted invalid pattern: "%s"')
+            '`%s` | `%s` matched but shouldn\'t')
 
         # Error states.
         error.add_error_code(
             'NO_TEST_PATTERNS', ErrorLevel.CRITICAL,
-            'leaf "%s" does not have any test cases')
+            '| leaf `%s` does not have any test cases')
         error.add_error_code(
             'UNRESTRICTED_TYPE', ErrorLevel.CRITICAL,
-            'leaf "%s" has unrestricted string type')
+            '| leaf `%s` has unrestricted string type')
 
 typedef_usage_stmt_regex = re.compile(r'([^\s:]+:)?([^\s:]+)')
 

pytests/tests/golden.txt

@@ -1,16 +1,16 @@
-testdata/python-plugin-test.yang:20: error: type "string" accepted invalid pattern: "ipv4"
-testdata/python-plugin-test.yang:21: error: type "string" rejected valid pattern: "ipv6"
-testdata/python-plugin-test.yang:23: error: leaf "ipv4-2" does not have any test cases
-testdata/python-plugin-test.yang:31: error: leaf "string" has unrestricted string type
-testdata/python-plugin-test.yang:34: error: leaf "union" has unrestricted string type
-testdata/python-plugin-test.yang:54: error: type "union" accepted invalid pattern: "ipv4"
-testdata/python-plugin-test.yang:55: error: type "union" rejected valid pattern: "ipv5"
-testdata/python-plugin-test.yang:64: error: type "t:ip-string" accepted invalid pattern: "ipv4"
-testdata/python-plugin-test.yang:65: error: type "t:ip-string" rejected valid pattern: "ipv5"
-testdata/python-plugin-test.yang:74: error: type "t:ip-string-typedef" accepted invalid pattern: "ipv4"
-testdata/python-plugin-test.yang:75: error: type "t:ip-string-typedef" rejected valid pattern: "ipv5"
-testdata/python-plugin-test.yang:91: error: type "union" accepted invalid pattern: "ipv4"
-testdata/python-plugin-test.yang:92: error: type "union" rejected valid pattern: "hehe"
-testdata/python-plugin-test.yang:93: error: type "union" accepted invalid pattern: "ipV5"
-testdata/python-plugin-test.yang:94: error: type "union" accepted invalid pattern: "ipv6"
-testdata/python-plugin-test.yang:104: error: type "t:ipv4-str" rejected valid pattern: "ipv6"
+| 20 | `string` | `ipv4` matched but shouldn't |
+| 21 | `string` | `ipv6` did not match |
+| 23 | | leaf `ipv4-2` does not have any test cases |
+| 31 | | leaf `string` has unrestricted string type |
+| 34 | | leaf `union` has unrestricted string type |
+| 54 | `union` | `ipv4` matched but shouldn't |
+| 55 | `union` | `ipv5` did not match |
+| 64 | `t:ip-string` | `ipv4` matched but shouldn't |
+| 65 | `t:ip-string` | `ipv5` did not match |
+| 74 | `t:ip-string-typedef` | `ipv4` matched but shouldn't |
+| 75 | `t:ip-string-typedef` | `ipv5` did not match |
+| 91 | `union` | `ipv4` matched but shouldn't |
+| 92 | `union` | `hehe` did not match |
+| 93 | `union` | `ipV5` matched but shouldn't |
+| 94 | `union` | `ipv6` matched but shouldn't |
+| 104 | `t:ipv4-str` | `ipv6` did not match |