From 6911da57d6b761bb8ef529224e00b284de7fe270 Mon Sep 17 00:00:00 2001 From: Andrew Richardson Date: Thu, 18 Aug 2022 13:24:12 -0400 Subject: [PATCH 1/4] Add deprecation warning for 'publicstorage' Signed-off-by: Andrew Richardson --- internal/orchestrator/orchestrator.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/orchestrator/orchestrator.go b/internal/orchestrator/orchestrator.go index 0daa5fc686..297a605ae2 100644 --- a/internal/orchestrator/orchestrator.go +++ b/internal/orchestrator/orchestrator.go @@ -439,6 +439,9 @@ func (or *orchestrator) initPlugins(ctx context.Context) (err error) { if ssType == "" { // Fallback and attempt to look for a "publicstorage" (deprecated) plugin ssType = config.GetString(coreconfig.PublicStorageType) + if ssType != "" { + log.L(ctx).Warnf("Your config uses the deprecated 'publicstorage' section name - please change to 'sharedstorage' instead") + } storageConfig = publicstorageConfig } if or.sharedstorage, err = ssfactory.GetPlugin(ctx, ssType); err != nil { From bc4e16fd83ad0a9500dc9115613272418bf369e8 Mon Sep 17 00:00:00 2001 From: Andrew Richardson Date: Thu, 18 Aug 2022 13:24:24 -0400 Subject: [PATCH 2/4] Update CLI version in manifest Signed-off-by: Andrew Richardson --- manifest.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifest.json b/manifest.json index 8f32bc12a2..b7b87c86a6 100644 --- a/manifest.json +++ b/manifest.json @@ -44,6 +44,6 @@ "release": "v1.0.1" }, "cli": { - "tag": "v1.0.1" + "tag": "v1.0.2" } } From 4830b4cf5d5a8860b33b4c1b535aad96aed28fe0 Mon Sep 17 00:00:00 2001 From: Andrew Richardson Date: Thu, 18 Aug 2022 13:28:58 -0400 Subject: [PATCH 3/4] Backport ffconfig tool with all 1.0.x migrations Signed-off-by: Andrew Richardson --- ffconfig/README.md | 25 ++++ ffconfig/main.go | 76 +++++++++++ ffconfig/migrate/config.go | 201 ++++++++++++++++++++++++++++ ffconfig/migrate/config_test.go | 45 +++++++ ffconfig/migrate/migrations.go | 91 +++++++++++++ ffconfig/migrate/migrations_test.go | 115 ++++++++++++++++ go.mod | 2 + go.sum | 3 + 8 files changed, 558 insertions(+) create mode 100644 ffconfig/README.md create mode 100644 ffconfig/main.go create mode 100644 ffconfig/migrate/config.go create mode 100644 ffconfig/migrate/config_test.go create mode 100644 ffconfig/migrate/migrations.go create mode 100644 ffconfig/migrate/migrations_test.go diff --git a/ffconfig/README.md b/ffconfig/README.md new file mode 100644 index 0000000000..40bc43ea56 --- /dev/null +++ b/ffconfig/README.md @@ -0,0 +1,25 @@ +# FireFly configuration tool + +A tool for managing and migrating config files for Hyperledger FireFly. + +## Installation + +If you have a local Go development environment, and you have included `${GOPATH}/bin` in your path, you can install with: + +```sh +go install github.com/hyperledger/firefly/ffconfig@latest +``` + +## Usage + +### Migration + +Parse a config file to find any deprecated items that should be updated to the latest config syntax. +The updated config will be written to stdout (or a file specified with `-o`). +``` +ffconfig migrate -f firefly.core.yml [-o new.yml] +``` + +You may optionally specify `--from` and `--to` versions to run a subset of the migrations. + +View the source code for all current migrations at [migrate/migrations.go](migrate/migrations.go). diff --git a/ffconfig/main.go b/ffconfig/main.go new file mode 100644 index 0000000000..ba5b2c45e3 --- /dev/null +++ b/ffconfig/main.go @@ -0,0 +1,76 @@ +// Copyright © 2022 Kaleido, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "io/ioutil" + "os" + + "github.com/hyperledger/firefly/ffconfig/migrate" + "github.com/spf13/cobra" +) + +var rootCmd = &cobra.Command{ + Use: "ffconfig", + Short: "FireFly configuration tool", + Long: "Tool for managing and migrating config files for Hyperledger FireFly", + RunE: func(cmd *cobra.Command, args []string) error { + return fmt.Errorf("a command is required") + }, +} + +var migrateCommand = &cobra.Command{ + Use: "migrate", + Short: "Migrate a config file to the current version", + RunE: func(cmd *cobra.Command, args []string) error { + cfg, err := ioutil.ReadFile(cfgFile) + if err != nil { + return err + } + out, err := migrate.Run(cfg, fromVersion, toVersion) + if err != nil { + return err + } + if outFile == "" { + fmt.Print(string(out)) + return nil + } + return ioutil.WriteFile(outFile, out, 0600) + }, +} + +var cfgFile string +var outFile string +var fromVersion string +var toVersion string + +func init() { + rootCmd.PersistentFlags().StringVarP(&cfgFile, "config", "f", "firefly.core.yml", "config file") + migrateCommand.PersistentFlags().StringVarP(&outFile, "out", "o", "", "output file (if unspecified, write to stdout)") + migrateCommand.PersistentFlags().StringVar(&fromVersion, "from", "", "from version (optional, such as 1.0.0)") + migrateCommand.PersistentFlags().StringVar(&toVersion, "to", "", "to version (optional, such as 1.1.0)") + rootCmd.AddCommand(migrateCommand) +} + +func main() { + if err := rootCmd.Execute(); err != nil { + fmt.Fprintf(os.Stderr, "%s\n", err) + os.Exit(1) + } + os.Exit(0) +} diff --git a/ffconfig/migrate/config.go b/ffconfig/migrate/config.go new file mode 100644 index 0000000000..f8c0a0c182 --- /dev/null +++ b/ffconfig/migrate/config.go @@ -0,0 +1,201 @@ +// Copyright © 2022 Kaleido, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package migrate + +import ( + "fmt" + "io" +) + +type ConfigItem struct { + value interface{} + parent *ConfigItem + name string + writer io.Writer +} + +type ConfigItemIterator struct { + items []*ConfigItem +} + +func NewConfigItem(value interface{}, writer io.Writer) *ConfigItem { + return &ConfigItem{value: value, writer: writer} +} + +func (c *ConfigItem) hasChild(name string) (exists bool) { + if v, ok := c.value.(map[interface{}]interface{}); ok { + _, exists = v[name] + } + return exists +} + +func (c *ConfigItem) deleteChild(name string) { + if v, ok := c.value.(map[interface{}]interface{}); ok { + delete(v, name) + } +} + +func (c *ConfigItem) setChild(name string, value interface{}) { + if v, ok := c.value.(map[interface{}]interface{}); ok { + v[name] = value + } +} + +func (c *ConfigItem) Get(name string) *ConfigItem { + if v, ok := c.value.(map[interface{}]interface{}); ok { + if child, ok := v[name]; ok { + return &ConfigItem{value: child, parent: c, name: name, writer: c.writer} + } + } + return &ConfigItem{value: nil, parent: c, name: name, writer: c.writer} +} + +func (c *ConfigItemIterator) Get(name string) *ConfigItemIterator { + items := make([]*ConfigItem, len(c.items)) + for i, item := range c.items { + items[i] = item.Get(name) + } + return &ConfigItemIterator{items: items} +} + +func (c *ConfigItem) Path() string { + if c.parent == nil || c.parent.name == "" { + return c.name + } + return c.parent.Path() + "." + c.name +} + +func (c *ConfigItem) Exists() bool { + return c.value != nil +} + +func (c *ConfigItem) Length() int { + if v, ok := c.value.([]interface{}); ok { + return len(v) + } + return 0 +} + +func (c *ConfigItem) Each() *ConfigItemIterator { + list, ok := c.value.([]interface{}) + if !ok { + return &ConfigItemIterator{items: make([]*ConfigItem, 0)} + } + items := make([]*ConfigItem, len(list)) + for i, val := range list { + items[i] = &ConfigItem{ + value: val, + parent: c.parent, + name: c.name, + writer: c.writer, + } + } + return &ConfigItemIterator{items: items} +} + +func (c *ConfigItemIterator) Run(fn func(item *ConfigItem)) *ConfigItemIterator { + for _, item := range c.items { + fn(item) + } + return c +} + +func (c *ConfigItem) Create() *ConfigItem { + if !c.Exists() { + if c.parent != nil { + c.parent.Create() + } + c.value = make(map[interface{}]interface{}) + c.parent.setChild(c.name, c.value) + } + return c +} + +func (c *ConfigItem) Set(value interface{}) *ConfigItem { + fmt.Fprintf(c.writer, "Create: %s: %s\n", c.Path(), value) + c.value = value + c.parent.Create() + c.parent.setChild(c.name, c.value) + return c +} + +func (c *ConfigItemIterator) Set(value interface{}) *ConfigItemIterator { + for _, item := range c.items { + item.Set(value) + } + return c +} + +func (c *ConfigItem) SetIfEmpty(value interface{}) *ConfigItem { + if !c.Exists() { + c.Set(value) + } + return c +} + +func (c *ConfigItem) Delete() *ConfigItem { + if c.Exists() { + fmt.Fprintf(c.writer, "Delete: %s\n", c.Path()) + c.parent.deleteChild(c.name) + } + return c +} + +func (c *ConfigItemIterator) Delete() *ConfigItemIterator { + for _, item := range c.items { + item.Delete() + } + return c +} + +func (c *ConfigItem) RenameTo(name string) *ConfigItem { + if c.Exists() { + if c.parent.hasChild(name) { + // Don't overwrite if the new key already exists + c.Delete() + } else { + fmt.Fprintf(c.writer, "Rename: %s -> .%s\n", c.Path(), name) + c.parent.deleteChild(c.name) + c.parent.setChild(name, c.value) + c.name = name + } + } + return c +} + +func (c *ConfigItemIterator) RenameTo(name string) *ConfigItemIterator { + for _, item := range c.items { + item.RenameTo(name) + } + return c +} + +func (c *ConfigItem) ReplaceValue(old interface{}, new interface{}) *ConfigItem { + if c.value == old { + fmt.Fprintf(c.writer, "Change: %s: %s -> %s\n", c.Path(), old, new) + c.value = new + c.parent.setChild(c.name, new) + } + return c +} + +func (c *ConfigItemIterator) ReplaceValue(old interface{}, new interface{}) *ConfigItemIterator { + for _, item := range c.items { + item.ReplaceValue(old, new) + } + return c +} diff --git a/ffconfig/migrate/config_test.go b/ffconfig/migrate/config_test.go new file mode 100644 index 0000000000..64ff66afae --- /dev/null +++ b/ffconfig/migrate/config_test.go @@ -0,0 +1,45 @@ +// Copyright © 2022 Kaleido, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package migrate + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestDeleteList(t *testing.T) { + value := map[interface{}]interface{}{ + "values": []interface{}{"test1", "test2"}, + } + config := &ConfigItem{value: value, writer: os.Stdout} + config.Get("values").Each().Delete() + assert.Equal(t, 0, len(value)) +} + +func TestNoRename(t *testing.T) { + value := map[interface{}]interface{}{ + "key1": "val1", + "key2": "val2", + } + config := &ConfigItem{value: value, writer: os.Stdout} + config.Get("key1").RenameTo("key2") + assert.Equal(t, map[interface{}]interface{}{ + "key2": "val2", + }, value) +} diff --git a/ffconfig/migrate/migrations.go b/ffconfig/migrate/migrations.go new file mode 100644 index 0000000000..652f603c46 --- /dev/null +++ b/ffconfig/migrate/migrations.go @@ -0,0 +1,91 @@ +// Copyright © 2022 Kaleido, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package migrate + +import ( + "fmt" + "os" + "strings" + + "github.com/blang/semver/v4" + "gopkg.in/yaml.v2" +) + +var migrations = map[string]func(root *ConfigItem){ + "1.0.0": func(root *ConfigItem) { + root.Get("org").Get("identity").RenameTo("key") + root.Get("tokens").Each().Get("connector").RenameTo("plugin").ReplaceValue("https", "fftokens") + root.Get("dataexchange").Get("type").ReplaceValue("https", "ffdx") + root.Get("dataexchange").Get("https").RenameTo("ffdx") + }, + + "1.0.3": func(root *ConfigItem) { + root.Get("dataexchange").Get("type").SetIfEmpty("ffdx") + }, + + "1.0.4": func(root *ConfigItem) { + root.Get("publicstorage").RenameTo("sharedstorage") + }, +} + +func getVersions() []semver.Version { + versions := make([]semver.Version, 0, len(migrations)) + for k := range migrations { + versions = append(versions, semver.MustParse(k)) + } + semver.Sort(versions) + return versions +} + +func migrateVersion(root *ConfigItem, version string) { + fmt.Fprintf(os.Stderr, "Version %s\n", version) + migrations[version](root) + fmt.Fprintln(os.Stderr) +} + +func Run(cfg []byte, fromVersion, toVersion string) (result []byte, err error) { + var from, to semver.Version + if fromVersion != "" { + if from, err = semver.Parse(strings.TrimPrefix(fromVersion, "v")); err != nil { + return nil, fmt.Errorf("bad 'from' version: %s", err) + } + } + if toVersion != "" { + if to, err = semver.Parse(strings.TrimPrefix(toVersion, "v")); err != nil { + return nil, fmt.Errorf("bad 'to' version: %s", err) + } + } + + data := make(map[interface{}]interface{}) + err = yaml.Unmarshal(cfg, &data) + if err != nil { + return nil, err + } + + root := NewConfigItem(data, os.Stderr) + for _, version := range getVersions() { + if fromVersion != "" && version.LT(from) { + continue + } + if toVersion != "" && version.GT(to) { + break + } + migrateVersion(root, version.String()) + } + + return yaml.Marshal(data) +} diff --git a/ffconfig/migrate/migrations_test.go b/ffconfig/migrate/migrations_test.go new file mode 100644 index 0000000000..ee67f8dfe3 --- /dev/null +++ b/ffconfig/migrate/migrations_test.go @@ -0,0 +1,115 @@ +// Copyright © 2022 Kaleido, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package migrate + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestMigrate1_0_0(t *testing.T) { + config := []byte(`database: + type: sqlite3 +dataexchange: {} +publicstorage: + type: ipfs +`) + result, err := Run(config, "", "1.0.0") + assert.NoError(t, err) + assert.Equal(t, `database: + type: sqlite3 +dataexchange: {} +publicstorage: + type: ipfs +`, string(result)) +} + +func TestMigratev1_0_0(t *testing.T) { + config := []byte(`database: + type: sqlite3 +dataexchange: {} +`) + result, err := Run(config, "", "v1.0.0") + assert.NoError(t, err) + assert.Equal(t, `database: + type: sqlite3 +dataexchange: {} +`, string(result)) +} + +func TestMigrate1_0_3(t *testing.T) { + config := []byte(`database: + type: sqlite3 +dataexchange: {} +`) + result, err := Run(config, "", "1.0.3") + assert.NoError(t, err) + assert.Equal(t, `database: + type: sqlite3 +dataexchange: + type: ffdx +`, string(result)) +} + +func TestMigrate1_0_3Only(t *testing.T) { + config := []byte(`database: + type: sqlite3 +dataexchange: {} +publicstorage: + type: ipfs +`) + result, err := Run(config, "1.0.2", "1.0.3") + assert.NoError(t, err) + assert.Equal(t, `database: + type: sqlite3 +dataexchange: + type: ffdx +publicstorage: + type: ipfs +`, string(result)) +} + +func TestMigrate1_0_4(t *testing.T) { + config := []byte(`database: + type: sqlite3 +dataexchange: {} +publicstorage: + type: ipfs +`) + result, err := Run(config, "", "1.0.4") + assert.NoError(t, err) + assert.Equal(t, `database: + type: sqlite3 +dataexchange: + type: ffdx +sharedstorage: + type: ipfs +`, string(result)) +} + +func TestMigrateBadVersions(t *testing.T) { + _, err := Run([]byte{}, "BAD!", "") + assert.Regexp(t, "bad 'from' version", err) + _, err = Run([]byte{}, "", "BAD!") + assert.Regexp(t, "bad 'to' version", err) +} + +func TestMigrateBadYAML(t *testing.T) { + _, err := Run([]byte{'\t'}, "", "") + assert.Regexp(t, "yaml: found character", err) +} diff --git a/go.mod b/go.mod index c8cb474f05..ecb2aa4610 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/Masterminds/squirrel v1.5.2 github.com/aidarkhanov/nanoid v1.0.8 github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59 // indirect + github.com/blang/semver/v4 v4.0.0 github.com/containerd/containerd v1.5.10 // indirect github.com/docker/go-units v0.4.0 github.com/getkin/kin-openapi v0.94.1-0.20220401165309-136a868a30c2 @@ -45,4 +46,5 @@ require ( golang.org/x/net v0.0.0-20220412020605-290c469a71a5 golang.org/x/text v0.3.7 gopkg.in/natefinch/lumberjack.v2 v2.0.0 + gopkg.in/yaml.v2 v2.4.0 ) diff --git a/go.sum b/go.sum index d39901416d..80ae94839f 100644 --- a/go.sum +++ b/go.sum @@ -171,7 +171,10 @@ github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngE github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4= github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/bshuster-repo/logrus-logstash-hook v0.4.1/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk= From 92c2fc118f1a4d88c3acd998db3df2f5dd8755df Mon Sep 17 00:00:00 2001 From: Andrew Richardson Date: Thu, 18 Aug 2022 13:41:39 -0400 Subject: [PATCH 4/4] Pin golangci-lint to v1.46.2 Signed-off-by: Andrew Richardson --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 9e3e515625..35e74f3d95 100644 --- a/Makefile +++ b/Makefile @@ -23,7 +23,7 @@ lint: ${LINT} ${MOCKERY}: $(VGO) install github.com/vektra/mockery/cmd/mockery@latest ${LINT}: - $(VGO) install github.com/golangci/golangci-lint/cmd/golangci-lint@latest + $(VGO) install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.46.2 ffcommon: $(eval WSCLIENT_PATH := $(shell $(VGO) list -f '{{.Dir}}' github.com/hyperledger/firefly-common/pkg/wsclient))