diff --git a/.github/workflows/demo.yml b/.github/workflows/demo.yml new file mode 100644 index 0000000..271bdc7 --- /dev/null +++ b/.github/workflows/demo.yml @@ -0,0 +1,63 @@ +name: Demo on openconfig/public + +on: + pull_request: + branches: [ main ] + +jobs: + + pattern: + name: pattern statement + runs-on: ubuntu-latest + strategy: + fail-fast: false + + steps: + - name: Check out code + uses: actions/checkout@v2 + + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: '3.x' + + - name: Set up pyang + run: pip3 install pyang + + - name: Get public repo + run: git clone https://github.com/openconfig/public.git ~/tmp/public + + - name: Demo output on openconfig/public + continue-on-error: true + run: | + OCDIR=~/tmp/public pytests/pattern_test.sh + + posix-pattern: + name: posix-pattern statement + runs-on: ubuntu-latest + strategy: + fail-fast: false + + steps: + - name: Set up Go 1.x + uses: actions/setup-go@v2 + # Use default go version so that we don't have to update it every time a new one comes out. + id: go + + - name: Check out code into the Go module directory + uses: actions/checkout@v2 + + - name: Get dependencies + run: | + go get -v -t -d ./... + + - name: Build + run: go build -v ./... + + - name: Get public repo + run: git clone https://github.com/openconfig/public.git ~/tmp/public + + - name: Demo output on openconfig/public + continue-on-error: true + run: | + go run gotests/main.go -model-root ~/tmp/public testdata/regexp-test.yang diff --git a/README.md b/README.md index e65e127..0adfda1 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,26 @@ # Pattern Statement Tests for OpenConfig YANG models -**Under implementation.** - Tests [pattern statement](https://tools.ietf.org/html/rfc7950#section-9.4.5) using a [pyang](https://github.com/mbj4668/pyang) plugin and [oc-ext:posix-pattern](https://github.com/openconfig/public/blob/master/release/models/openconfig-extensions.yang#L114) using [goyang](https://github.com/openconfig/goyang). +## Demo CI Workflow + +There is a demo CI workflow that runs on Pull Requests. They are used to demo +the result of running the tests on the current YANG models. If they cause an +expected test failure (perhaps because you added an uncaught corner case), it +will not block merge and a minor version increment will be given. + +## Releases + +Releases are synchronized with the current OpenConfig YANG models. If new +breaking tests are added (e.g. test cases handled incorrectly by a current +pattern regex), then the minor version must be incremented. + +At this time, major version updates are not anticipated, but could occur as a +result of major changes to the repository. + -------------------------------------------------------------------------------- [OpenConfig YANG models](https://github.com/openconfig/public/blob/master/README.md) diff --git a/go.mod b/go.mod index 201d309..78d8acb 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,8 @@ go 1.15 require ( github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b - github.com/openconfig/gnmi v0.0.0-20201217212801-57b8e7af2d36 + github.com/google/go-cmp v0.5.0 + github.com/openconfig/gnmi v0.0.0-20201217212801-57b8e7af2d36 // indirect github.com/openconfig/goyang v0.2.4 github.com/openconfig/ygot v0.9.0 ) diff --git a/gotests/main.go b/gotests/main.go index d040929..807e025 100644 --- a/gotests/main.go +++ b/gotests/main.go @@ -19,10 +19,11 @@ package main import ( "flag" + "fmt" + "os" log "github.com/golang/glog" "github.com/openconfig/pattern-regex-tests/gotests/patterncheck" - "github.com/openconfig/ygot/util" ) var ( @@ -32,17 +33,25 @@ var ( func main() { flag.Parse() + code := 0 + defer func() { + os.Exit(code) + }() + if *modelRoot == "" { log.Error("Must supply model-root path") } - if err := patterncheck.CheckRegexps(flag.Args(), []string{*modelRoot}); err != nil { - if errors, ok := err.(util.Errors); ok { - for _, err := range errors { - log.Errorln(err) - } - } else { - log.Exit(err) + failureMessages, err := patterncheck.CheckRegexps(flag.Args(), []string{*modelRoot}) + if err != nil { + log.Exit(err) + } + if len(failureMessages) != 0 { + code = 1 + fmt.Fprintln(os.Stderr, "| leaf | typedef | error |") + fmt.Fprintln(os.Stderr, "| --- | --- | --- |") + for _, msg := range failureMessages { + fmt.Fprintln(os.Stderr, msg) } } } diff --git a/gotests/patterncheck/patterncheck.go b/gotests/patterncheck/patterncheck.go index 625c33a..ef70a70 100644 --- a/gotests/patterncheck/patterncheck.go +++ b/gotests/patterncheck/patterncheck.go @@ -26,8 +26,8 @@ import ( ) var ( - passCaseExt = regexp.MustCompile(`\w:pattern-test-pass`) - failCaseExt = regexp.MustCompile(`\w:pattern-test-fail`) + passCaseExt = regexp.MustCompile(`\w+:pattern-test-pass`) + failCaseExt = regexp.MustCompile(`\w+:pattern-test-fail`) ) // YANGLeaf is a structure used to describe a particular leaf of YANG schema. @@ -45,39 +45,42 @@ type RegexpTest struct { // CheckRegexps tests mock input data against a set of leaves that have pattern // test cases specified for them. It ensures that the regexp compiles as a // POSIX regular expression according to the OpenConfig style guide. -func CheckRegexps(yangfiles, paths []string) error { +func CheckRegexps(yangfiles, paths []string) ([]string, error) { yangE, errs := yangutil.ProcessModules(yangfiles, paths) if len(errs) != 0 { - return fmt.Errorf("could not parse modules: %v", errs) + return nil, fmt.Errorf("could not parse modules: %v", errs) } if len(yangE) == 0 { - return fmt.Errorf("did not parse any modules") + return nil, fmt.Errorf("did not parse any modules") } - var errs2 util.Errors + var patternErrs util.Errors + var allFailMessages []string for _, mod := range yangE { for _, entry := range mod.Dir { - if err := checkEntryPatterns(entry); err != nil { - errs2 = util.AppendErr(errs2, err) + if failMessages, err := checkEntryPatterns(entry); err != nil { + patternErrs = util.AppendErr(patternErrs, err) + } else { + allFailMessages = append(allFailMessages, failMessages...) } } } - if len(errs2) == 0 { - return nil + if len(patternErrs) != 0 { + return nil, patternErrs } - return errs2 + return allFailMessages, nil } -func checkEntryPatterns(entry *yang.Entry) error { +func checkEntryPatterns(entry *yang.Entry) ([]string, error) { if entry.Kind != yang.LeafEntry { - return nil + return nil, nil } if len(entry.Errors) != 0 { - return fmt.Errorf("entry had associated errors: %v", entry.Errors) + return nil, fmt.Errorf("entry had associated errors: %v", entry.Errors) } - var errs util.Errors + var failMessages []string for _, ext := range entry.Exts { var wantMatch bool switch { @@ -93,7 +96,7 @@ func checkEntryPatterns(entry *yang.Entry) error { if len(entry.Type.Type) == 0 { var err error if gotMatch, err = checkPatterns(ext.Argument, entry.Type.POSIXPattern); err != nil { - return err + return nil, err } } else { // Handle unions. @@ -107,26 +110,23 @@ func checkEntryPatterns(entry *yang.Entry) error { } matches, err := checkPatterns(ext.Argument, membertype.POSIXPattern) if err != nil { - return err + return nil, err } gotMatch = gotMatch || matches } } - matchDesc := fmt.Sprintf("%q doesn't match type %s (leaf %s)", ext.Argument, entry.Type.Name, entry.Name) - if gotMatch { - matchDesc = fmt.Sprintf("%q matches type %s (leaf %s)", ext.Argument, entry.Type.Name, entry.Name) + matchDesc := fmt.Sprintf("| `%s` | `%s` | `%s` matched but shouldn't |", entry.Name, entry.Type.Name, ext.Argument) + if !gotMatch { + matchDesc = fmt.Sprintf("| `%s` | `%s` | `%s` did not match |", entry.Name, entry.Type.Name, ext.Argument) } if gotMatch != wantMatch { - errs = util.AppendErr(errs, fmt.Errorf("fail: %s", matchDesc)) + failMessages = append(failMessages, matchDesc) } log.Infof("pass: %s", matchDesc) } - if len(errs) == 0 { - return nil - } - return errs + return failMessages, nil } // checkPatterns compiles all given POSIX patterns, and returns true if diff --git a/gotests/patterncheck/patterncheck_test.go b/gotests/patterncheck/patterncheck_test.go index ecad1ea..a03169e 100644 --- a/gotests/patterncheck/patterncheck_test.go +++ b/gotests/patterncheck/patterncheck_test.go @@ -17,7 +17,7 @@ package patterncheck import ( "testing" - "github.com/openconfig/gnmi/errdiff" + "github.com/google/go-cmp/cmp" ) func TestCheckRegexps(t *testing.T) { @@ -25,32 +25,44 @@ func TestCheckRegexps(t *testing.T) { desc string inFiles []string inPaths []string - wantErrSubstring string + wantFailMessages []string }{{ desc: "passing cases", inFiles: []string{"testdata/passing.yang"}, inPaths: []string{"../../testdata"}, }, { - desc: "simple leaf fail", - inFiles: []string{"testdata/simple-leaf-fail.yang"}, - inPaths: []string{"../../testdata"}, - wantErrSubstring: `"ipv4" matches type string (leaf ipv-0), fail: "ipv6" doesn't match type string (leaf ipv-0)`, + desc: "simple leaf fail", + inFiles: []string{"testdata/simple-leaf-fail.yang"}, + inPaths: []string{"../../testdata"}, + wantFailMessages: []string{ + "| `ipv-0` | `string` | `ipv4` matched but shouldn't |", + "| `ipv-0` | `string` | `ipv6` did not match |", + }, }, { - desc: "union leaf fail", - inFiles: []string{"testdata/union-leaf-fail.yang"}, - inPaths: []string{"../../testdata"}, - wantErrSubstring: `fail: "ipv4" matches type ip-string-typedef (leaf ipv-0), fail: "ipv5" doesn't match type ip-string-typedef (leaf ipv-0)`, + desc: "union leaf fail", + inFiles: []string{"testdata/union-leaf-fail.yang"}, + inPaths: []string{"../../testdata"}, + wantFailMessages: []string{ + "| `ipv-0` | `ip-string-typedef` | `ipv4` matched but shouldn't |", + "| `ipv-0` | `ip-string-typedef` | `ipv5` did not match |", + }, }, { - desc: "derived string type fail", - inFiles: []string{"testdata/derived-string-fail.yang"}, - inPaths: []string{"../../testdata"}, - wantErrSubstring: `fail: "ipV4" doesn't match type ipv4-address-str (leaf ipv-0), fail: "ipV4-address" matches type ipv4-address-str (leaf ipv-0)`, + desc: "derived string type fail", + inFiles: []string{"testdata/derived-string-fail.yang"}, + inPaths: []string{"../../testdata"}, + wantFailMessages: []string{ + "| `ipv-0` | `ipv4-address-str` | `ipV4` did not match |", + "| `ipv-0` | `ipv4-address-str` | `ipV4-address` matched but shouldn't |", + }, }} for _, tt := range tests { t.Run(tt.desc, func(t *testing.T) { - got := CheckRegexps(tt.inFiles, tt.inPaths) - if diff := errdiff.Substring(got, tt.wantErrSubstring); diff != "" { + got, err := CheckRegexps(tt.inFiles, tt.inPaths) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(got, tt.wantFailMessages); diff != "" { t.Errorf("(-got, +want):\n%s", diff) } }) diff --git a/pytests/pattern_test.sh b/pytests/pattern_test.sh index 69ec26f..22a7dc6 100755 --- a/pytests/pattern_test.sh +++ b/pytests/pattern_test.sh @@ -9,4 +9,13 @@ fi TEST_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" REPO_DIR="$TEST_DIR/.." -pyang -p $OCDIR -p "$REPO_DIR/testdata" --msg-template="line {line}: {msg}" --plugindir "$REPO_DIR/pytests/plugins" --check-patterns "$REPO_DIR/testdata/regexp-test.yang" +tmpstderr=$(mktemp) +pyang -p $OCDIR -p "$REPO_DIR/testdata" --msg-template="| {line} | {msg} |" --plugindir "$REPO_DIR/pytests/plugins" --check-patterns "$REPO_DIR/testdata/regexp-test.yang" 2> $tmpstderr +retcode=$? +if [ $retcode -ne 0 ]; then + >&2 echo "| Line # | typedef | error |" + >&2 echo "| --- | --- | --- |" +fi +>&2 cat $tmpstderr +rm $tmpstderr +exit $retcode diff --git a/pytests/plugins/pattern_test.py b/pytests/plugins/pattern_test.py index 5cfc525..91c166f 100644 --- a/pytests/plugins/pattern_test.py +++ b/pytests/plugins/pattern_test.py @@ -57,18 +57,18 @@ def setup_ctx(self, ctx): # Test case failure states. error.add_error_code( 'VALID_PATTERN_DOES_NOT_MATCH', ErrorLevel.MAJOR, - 'type "%s" rejected valid pattern: "%s"') + '`%s` | `%s` did not match') error.add_error_code( 'INVALID_PATTERN_MATCH', ErrorLevel.MAJOR, - 'type "%s" accepted invalid pattern: "%s"') + '`%s` | `%s` matched but shouldn\'t') # Error states. error.add_error_code( 'NO_TEST_PATTERNS', ErrorLevel.CRITICAL, - 'leaf "%s" does not have any test cases') + '| leaf `%s` does not have any test cases') error.add_error_code( 'UNRESTRICTED_TYPE', ErrorLevel.CRITICAL, - 'leaf "%s" has unrestricted string type') + '| leaf `%s` has unrestricted string type') typedef_usage_stmt_regex = re.compile(r'([^\s:]+:)?([^\s:]+)') diff --git a/pytests/tests/golden.txt b/pytests/tests/golden.txt index 27e9060..f977069 100644 --- a/pytests/tests/golden.txt +++ b/pytests/tests/golden.txt @@ -1,16 +1,16 @@ -testdata/python-plugin-test.yang:20: error: type "string" accepted invalid pattern: "ipv4" -testdata/python-plugin-test.yang:21: error: type "string" rejected valid pattern: "ipv6" -testdata/python-plugin-test.yang:23: error: leaf "ipv4-2" does not have any test cases -testdata/python-plugin-test.yang:31: error: leaf "string" has unrestricted string type -testdata/python-plugin-test.yang:34: error: leaf "union" has unrestricted string type -testdata/python-plugin-test.yang:54: error: type "union" accepted invalid pattern: "ipv4" -testdata/python-plugin-test.yang:55: error: type "union" rejected valid pattern: "ipv5" -testdata/python-plugin-test.yang:64: error: type "t:ip-string" accepted invalid pattern: "ipv4" -testdata/python-plugin-test.yang:65: error: type "t:ip-string" rejected valid pattern: "ipv5" -testdata/python-plugin-test.yang:74: error: type "t:ip-string-typedef" accepted invalid pattern: "ipv4" -testdata/python-plugin-test.yang:75: error: type "t:ip-string-typedef" rejected valid pattern: "ipv5" -testdata/python-plugin-test.yang:91: error: type "union" accepted invalid pattern: "ipv4" -testdata/python-plugin-test.yang:92: error: type "union" rejected valid pattern: "hehe" -testdata/python-plugin-test.yang:93: error: type "union" accepted invalid pattern: "ipV5" -testdata/python-plugin-test.yang:94: error: type "union" accepted invalid pattern: "ipv6" -testdata/python-plugin-test.yang:104: error: type "t:ipv4-str" rejected valid pattern: "ipv6" +| 20 | `string` | `ipv4` matched but shouldn't | +| 21 | `string` | `ipv6` did not match | +| 23 | | leaf `ipv4-2` does not have any test cases | +| 31 | | leaf `string` has unrestricted string type | +| 34 | | leaf `union` has unrestricted string type | +| 54 | `union` | `ipv4` matched but shouldn't | +| 55 | `union` | `ipv5` did not match | +| 64 | `t:ip-string` | `ipv4` matched but shouldn't | +| 65 | `t:ip-string` | `ipv5` did not match | +| 74 | `t:ip-string-typedef` | `ipv4` matched but shouldn't | +| 75 | `t:ip-string-typedef` | `ipv5` did not match | +| 91 | `union` | `ipv4` matched but shouldn't | +| 92 | `union` | `hehe` did not match | +| 93 | `union` | `ipV5` matched but shouldn't | +| 94 | `union` | `ipv6` matched but shouldn't | +| 104 | `t:ipv4-str` | `ipv6` did not match | diff --git a/pytests/tests/plugin_test.sh b/pytests/tests/plugin_test.sh index ec6dceb..02d9e13 100755 --- a/pytests/tests/plugin_test.sh +++ b/pytests/tests/plugin_test.sh @@ -5,7 +5,7 @@ REPO_DIR="$TEST_DIR/../.." cwd=$PWD cd $TEST_DIR -pyang -p "$REPO_DIR/testdata" -p "testdata" --plugindir "$REPO_DIR/pytests/plugins" --check-patterns "testdata/python-plugin-test.yang" 2>&1 | diff - "golden.txt" +pyang -p "$REPO_DIR/testdata" -p "testdata" --msg-template="| {line} | {msg} |" --plugindir "$REPO_DIR/pytests/plugins" --check-patterns "testdata/python-plugin-test.yang" 2>&1 | diff - "golden.txt" retcode=$? cd $cwd diff --git a/testdata/regexp-test.yang b/testdata/regexp-test.yang index b910c44..36940e5 100644 --- a/testdata/regexp-test.yang +++ b/testdata/regexp-test.yang @@ -16,6 +16,20 @@ module regexp-test { pt:pattern-test-fail "1.1.1.256"; pt:pattern-test-fail "256.1.1.1%eth0"; } + leaf ipv4-address-zoned { + type ocinet:ipv4-address-zoned; + pt:pattern-test-pass "255.1.1.1%eth0"; + pt:pattern-test-pass "255.255.255.255%eth1"; + pt:pattern-test-pass "0.0.0.0%PoRt3"; + pt:pattern-test-pass "1.1.1.1%FOX10_mouse5"; + pt:pattern-test-fail "255.1.1.1%"; + pt:pattern-test-fail "255.255.255.255"; + pt:pattern-test-fail "0.0.0.0"; + pt:pattern-test-fail "1.1.1.1"; + pt:pattern-test-fail "256.255.255.255"; + pt:pattern-test-fail "1.1.1.256"; + pt:pattern-test-fail "256.1.1.1%eth0"; + } leaf ip-address { type ocinet:ip-address; pt:pattern-test-pass "255.255.255.255";