Public release of Skycfg, a library for building complex typed configs.

.gitattributes

@@ -0,0 +1 @@
+*.sky linguist-language=Python

.travis.yml

@@ -0,0 +1,25 @@
+language: go
+go_import_path: github.com/stripe/skycfg
+go:
+  - "1.11"
+  - master
+env:
+  global:
+    - PROTOC_VERSION=3.6.1
+    - GO111MODULE=on
+
+install:
+  - mkdir -p "${TRAVIS_HOME}/third_party/protobuf-${PROTOC_VERSION}"
+  - |
+    (
+      cd "${TRAVIS_HOME}/third_party/protobuf-${PROTOC_VERSION}"
+      wget "https://github.com/google/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-${TRAVIS_OS_NAME}-x86_64.zip"
+      unzip "protoc-${PROTOC_VERSION}-${TRAVIS_OS_NAME}-x86_64.zip"
+    )
+  - export PATH="${TRAVIS_HOME}/third_party/protobuf-${PROTOC_VERSION}/bin:${PATH}"
+  - go get github.com/golang/protobuf/protoc-gen-go
+  - protoc --go_out="${TRAVIS_HOME}/gopath/src" --proto_path=testdata test_proto_v2.proto test_proto_v3.proto
+  - go get -t -v ./...
+
+script:
+  - go test -v ./...

AUTHORS

@@ -0,0 +1,10 @@
+# This is the list of Skycfg authors for copyright purposes.
+#
+# This does not necessarily list everyone who has contributed code, since in
+# some cases, their employer may be the copyright holder.  To see the full list
+# of contributors, see the revision history in source control.
+
+# Names of people must include a contact email, in the format:
+#     Name <email address>
+
+Stripe, Inc.

CONTRIBUTORS

@@ -0,0 +1,13 @@
+# The AUTHORS file lists the copyright holders; this file
+# lists people.  For example, Stripe employees are listed here
+# but not in AUTHORS, because Stripe holds the copyright.
+#
+# Names should be added to this file as:
+#     Name <email address>
+
+Alexander Pakulov <apakulov@stripe.com>
+Benjamin Yolken <yolken@stripe.com>
+Isaac Diamond <idiamond@stripe.com>
+John Millikin <jmillikin@stripe.com>
+
+# Please alphabetize new entries.

README.md

@@ -0,0 +1,17 @@
+# Skycfg
+
+https://github.com/golang/go/wiki/Modules
+
+```
+$ go version
+go version go1.11 darwin/amd64
+$ export GO111MODULE=on
+$ go build ./...
+[...]
+$ godoc -http 'localhost:8080'
+```
+
+```
+$ protoc --go_out="${GOPATH}/src" --proto_path=testdata test_proto_v2.proto test_proto_v3.proto
+$ go test -v ./...
+```

go.mod

@@ -0,0 +1,12 @@
+module github.com/stripe/skycfg
+
+require (
+	github.com/golang/protobuf v1.2.0
+	github.com/google/skylark v0.0.0-20181005144900-2320ce69c4b8
+	github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348
+	golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3 // indirect
+	golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f // indirect
+	gopkg.in/yaml.v2 v2.2.1
+)
+
+replace github.com/kylelemons/godebug => github.com/jmillikin-stripe/godebug v0.0.0-20180620173319-8279e1966bc1

go.sum

@@ -0,0 +1,18 @@
+github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/google/skylark v0.0.0-20180918192949-ea6a6cb3d5aa h1:9KHAqoD+4GbuXeWOioTA0vw2VeeZku1j+bJRyRtHb6E=
+github.com/google/skylark v0.0.0-20180918192949-ea6a6cb3d5aa/go.mod h1:CKSX6SxHW1vp20ZNaeGe3TFFBIwCG6vaYrpAiOzX+NA=
+github.com/google/skylark v0.0.0-20181005144900-2320ce69c4b8 h1:yZrNpuy1q9hynqBgg8nZCtYRdoKFQhIRcECVK+KxPnE=
+github.com/google/skylark v0.0.0-20181005144900-2320ce69c4b8/go.mod h1:CKSX6SxHW1vp20ZNaeGe3TFFBIwCG6vaYrpAiOzX+NA=
+github.com/jmillikin-stripe/godebug v0.0.0-20180620173319-8279e1966bc1 h1:JgsVrDAUy59N248f3l4RGZ0hij5u1HTit8iJr1mFSBY=
+github.com/jmillikin-stripe/godebug v0.0.0-20180620173319-8279e1966bc1/go.mod h1:gFqr/IKD8P+Hluq9gThCR944BAu6jUqd5H/R3PrPfuM=
+github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3vkb4ax0b5D2DHbNAUsen0Gx5wZoq3lV4=
+github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
+golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3 h1:czFLhve3vsQetD6JOJ8NZZvGQIXlnN3/yXxbT6/awxI=
+golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

go/skycfg/json.go

@@ -0,0 +1,144 @@
+package skycfg
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+
+	"github.com/google/skylark"
+	yaml "gopkg.in/yaml.v2"
+)
+
+// jsonModule returns a Skylark module for JSON helpers.
+func jsonModule() skylark.Value {
+	return &skyModule{
+		name: "json",
+		attrs: skylark.StringDict{
+			"marshal": jsonMarshal(),
+		},
+	}
+}
+
+// jsonMarshal returns a Skylark function for marshaling plain values
+// (dicts, lists, etc) to JSON.
+//
+//  def json.marshal(value) -> str
+func jsonMarshal() skylark.Callable {
+	return skylark.NewBuiltin("json.marshal", fnJsonMarshal)
+}
+
+func fnJsonMarshal(t *skylark.Thread, fn *skylark.Builtin, args skylark.Tuple, kwargs []skylark.Tuple) (skylark.Value, error) {
+	var v skylark.Value
+	if err := skylark.UnpackArgs(fn.Name(), args, kwargs, "value", &v); err != nil {
+		return nil, err
+	}
+	var buf bytes.Buffer
+	if err := writeJSON(&buf, v); err != nil {
+		return nil, err
+	}
+	return skylark.String(buf.String()), nil
+}
+
+// yamlModule returns a Skylark module for YAML helpers.
+func yamlModule() skylark.Value {
+	return &skyModule{
+		name: "yaml",
+		attrs: skylark.StringDict{
+			"marshal": yamlMarshal(),
+		},
+	}
+}
+
+// yamlMarshal returns a Skylark function for marshaling plain values
+// (dicts, lists, etc) to YAML.
+//
+//  def yaml.marshal(value) -> str
+func yamlMarshal() skylark.Callable {
+	return skylark.NewBuiltin("yaml.marshal", fnYamlMarshal)
+}
+
+func fnYamlMarshal(t *skylark.Thread, fn *skylark.Builtin, args skylark.Tuple, kwargs []skylark.Tuple) (skylark.Value, error) {
+	var v skylark.Value
+	if err := skylark.UnpackArgs(fn.Name(), args, kwargs, "value", &v); err != nil {
+		return nil, err
+	}
+	var buf bytes.Buffer
+	if err := writeJSON(&buf, v); err != nil {
+		return nil, err
+	}
+	var jsonObj interface{}
+	if err := yaml.Unmarshal(buf.Bytes(), &jsonObj); err != nil {
+		return nil, err
+	}
+	yamlBytes, err := yaml.Marshal(jsonObj)
+	if err != nil {
+		return nil, err
+	}
+	return skylark.String(yamlBytes), nil
+}
+
+// Adapted from struct-specific JSON function:
+// https://github.com/google/skylark/blob/67717b5898061eb621519a94a4b89cedede9bca0/skylarkstruct/struct.go#L321
+func writeJSON(out *bytes.Buffer, v skylark.Value) error {
+	switch v := v.(type) {
+	case skylark.NoneType:
+		out.WriteString("null")
+	case skylark.Bool:
+		fmt.Fprintf(out, "%t", v)
+	case skylark.Int:
+		out.WriteString(v.String())
+	case skylark.Float:
+		fmt.Fprintf(out, "%g", v)
+	case skylark.String:
+		s := string(v)
+		if goQuoteIsSafe(s) {
+			fmt.Fprintf(out, "%q", s)
+		} else {
+			// vanishingly rare for text strings
+			data, _ := json.Marshal(s)
+			out.Write(data)
+		}
+	case skylark.Indexable: // Tuple, List
+		out.WriteByte('[')
+		for i, n := 0, skylark.Len(v); i < n; i++ {
+			if i > 0 {
+				out.WriteString(", ")
+			}
+			if err := writeJSON(out, v.Index(i)); err != nil {
+				return err
+			}
+		}
+		out.WriteByte(']')
+	case *skylark.Dict:
+		out.WriteByte('{')
+		for i, itemPair := range v.Items() {
+			key := itemPair[0]
+			value := itemPair[1]
+			if i > 0 {
+				out.WriteString(", ")
+			}
+			if err := writeJSON(out, key); err != nil {
+				return err
+			}
+			out.WriteString(": ")
+			if err := writeJSON(out, value); err != nil {
+				return err
+			}
+		}
+		out.WriteByte('}')
+	default:
+		return fmt.Errorf("cannot convert %s to JSON", v.Type())
+	}
+	return nil
+}
+
+func goQuoteIsSafe(s string) bool {
+	for _, r := range s {
+		// JSON doesn't like Go's \xHH escapes for ASCII control codes,
+		// nor its \UHHHHHHHH escapes for runes >16 bits.
+		if r < 0x20 || r >= 0x10000 {
+			return false
+		}
+	}
+	return true
+}

go/skycfg/json_test.go

@@ -0,0 +1,115 @@
+package skycfg
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/google/skylark"
+)
+
+type TestCase struct {
+	skyExpr   string
+	expOutput string
+}
+
+func TestSkyToJson(t *testing.T) {
+	thread := new(skylark.Thread)
+	env := skylark.StringDict{
+		"json": jsonModule(),
+	}
+
+	testCases := []TestCase{
+		TestCase{
+			skyExpr:   "123",
+			expOutput: "123",
+		},
+		TestCase{
+			skyExpr:   `{"a": 5, 13: 2, "k": {"k2": "v"}}`,
+			expOutput: `{"a": 5, 13: 2, "k": {"k2": "v"}}`,
+		},
+		TestCase{
+			skyExpr:   `[1, 2, 3, "abc", None, 15, True, False, {"k": "v"}]`,
+			expOutput: `[1, 2, 3, "abc", null, 15, true, false, {"k": "v"}]`,
+		},
+	}
+
+	for _, testCase := range testCases {
+		v, err := skylark.Eval(
+			thread,
+			"<expr>",
+			fmt.Sprintf("json.marshal(%s)", testCase.skyExpr),
+			env,
+		)
+		if err != nil {
+			t.Error("Error from eval", "\nExpected nil", "\nGot", err)
+		}
+		exp := skylark.String(testCase.expOutput)
+		if v != exp {
+			t.Error(
+				"Bad return value from json.marshal",
+				"\nExpected",
+				exp,
+				"\nGot",
+				v,
+			)
+		}
+	}
+}
+
+func TestSkyToYaml(t *testing.T) {
+	thread := new(skylark.Thread)
+	env := skylark.StringDict{
+		"yaml": yamlModule(),
+	}
+
+	testCases := []TestCase{
+		TestCase{
+			skyExpr: "123",
+			expOutput: `123
+`,
+		},
+		TestCase{
+			skyExpr: `{"a": 5, 13: 2, "k": {"k2": "v"}}`,
+			expOutput: `13: 2
+a: 5
+k:
+  k2: v
+`,
+		},
+		TestCase{
+			skyExpr: `[1, 2, 3, "abc", None, 15, True, False, {"k": "v"}]`,
+			expOutput: `- 1
+- 2
+- 3
+- abc
+- null
+- 15
+- true
+- false
+- k: v
+`,
+		},
+	}
+
+	for _, testCase := range testCases {
+		v, err := skylark.Eval(
+			thread,
+			"<expr>",
+			fmt.Sprintf("yaml.marshal(%s)", testCase.skyExpr),
+			env,
+		)
+		if err != nil {
+			t.Error("Error from eval", "\nExpected nil", "\nGot", err)
+		}
+		exp := skylark.String(testCase.expOutput)
+		if v != exp {
+			t.Error(
+				"Bad return value from yaml.marshal",
+				"\nExpected",
+				exp,
+				"\nGot",
+				v,
+			)
+		}
+	}
+}