diff --git a/cmd/application.go b/cmd/application.go index f12a967..e3db56e 100644 --- a/cmd/application.go +++ b/cmd/application.go @@ -1,4 +1,4 @@ -// Copyright © 2020 The Things Network Foundation, The Things Industries B.V. +// Copyright © 2023 The Things Network Foundation, The Things Industries B.V. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -19,18 +19,16 @@ import ( "go.thethings.network/lorawan-stack-migrate/pkg/source" ) -var ( - applicationsCmd = &cobra.Command{ - Use: "application [app-id] ...", - Aliases: []string{"applications", "app"}, - Short: "Export all devices of an application", - RunE: func(cmd *cobra.Command, args []string) error { - return exportCommand(cmd, args, func(s source.Source, item string) error { - return s.RangeDevices(item, exportCfg.exportDev) - }) - }, - } -) +var applicationsCmd = &cobra.Command{ + Use: "application [app-id] ...", + Aliases: []string{"applications", "app"}, + Short: "Export all devices of an application", + RunE: func(cmd *cobra.Command, args []string) error { + return exportCommand(cmd, args, func(s source.Source, item string) error { + return s.RangeDevices(item, exportCfg.exportDev) + }) + }, +} func init() { applicationsCmd.Flags().AddFlagSet(source.FlagSet()) diff --git a/cmd/devices.go b/cmd/devices.go index 2113f12..c48e980 100644 --- a/cmd/devices.go +++ b/cmd/devices.go @@ -1,4 +1,4 @@ -// Copyright © 2020 The Things Network Foundation, The Things Industries B.V. +// Copyright © 2023 The Things Network Foundation, The Things Industries B.V. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -19,16 +19,14 @@ import ( "go.thethings.network/lorawan-stack-migrate/pkg/source" ) -var ( - devicesCmd = &cobra.Command{ - Use: "device [dev-id] ...", - Short: "Export devices by DevEUI", - Aliases: []string{"end-devices", "end-device", "devices", "dev"}, - RunE: func(cmd *cobra.Command, args []string) error { - return exportCommand(cmd, args, exportCfg.exportDev) - }, - } -) +var devicesCmd = &cobra.Command{ + Use: "device [dev-id] ...", + Short: "Export devices by DevEUI", + Aliases: []string{"end-devices", "end-device", "devices", "dev"}, + RunE: func(cmd *cobra.Command, args []string) error { + return exportCommand(cmd, args, exportCfg.exportDev) + }, +} func init() { devicesCmd.Flags().AddFlagSet(source.FlagSet()) diff --git a/cmd/export.go b/cmd/export.go index 42833ea..a960844 100644 --- a/cmd/export.go +++ b/cmd/export.go @@ -1,4 +1,4 @@ -// Copyright © 2020 The Things Network Foundation, The Things Industries B.V. +// Copyright © 2023 The Things Network Foundation, The Things Industries B.V. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -31,7 +31,7 @@ func exportCommand(cmd *cobra.Command, args []string, f func(s source.Source, it iter = NewListIterator(args) } - s, err := source.NewSource(ctx, cmd.Flags()) + s, err := source.NewSource(ctx) if err != nil { return err } diff --git a/cmd/root.go b/cmd/root.go index 91d83a2..b5e65c7 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -1,4 +1,4 @@ -// Copyright © 2020 The Things Network Foundation, The Things Industries B.V. +// Copyright © 2023 The Things Network Foundation, The Things Industries B.V. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -19,6 +19,7 @@ import ( "os" "github.com/spf13/cobra" + "go.thethings.network/lorawan-stack-migrate/pkg/source" "go.thethings.network/lorawan-stack/v3/pkg/log" "go.thethings.network/lorawan-stack/v3/pkg/rpcmiddleware/rpclog" ) @@ -26,6 +27,7 @@ import ( var ( logger *log.Logger ctx context.Context + rootCfg = &source.RootConfig exportCfg = exportConfig{} rootCmd = &cobra.Command{ Use: "ttn-lw-migrate", @@ -66,9 +68,16 @@ func Execute() int { } func init() { - rootCmd.PersistentFlags().Bool("verbose", false, "Verbose output") - rootCmd.PersistentFlags().Bool("dry-run", false, "Do everything except resetting root and session keys of exported devices") - rootCmd.PersistentFlags().String("frequency-plans-url", "https://raw.githubusercontent.com/TheThingsNetwork/lorawan-frequency-plans/master", "URL for fetching frequency plans") - rootCmd.PersistentFlags().Bool("set-eui-as-id", false, "Use the DevEUI as ID") - rootCmd.PersistentFlags().String("dev-id-prefix", "", "(optional) value to be prefixed to the resulting device IDs") + rootCmd.PersistentFlags().BoolVar(&rootCfg.DryRun, + "dry-run", + false, + "Do everything except resetting root and session keys of exported devices") + rootCmd.PersistentFlags().BoolVar(&rootCfg.Verbose, + "verbose", + false, + "Verbose output") + rootCmd.PersistentFlags().StringVar(&rootCfg.FrequencyPlansURL, + "frequency-plans-url", + "https://raw.githubusercontent.com/TheThingsNetwork/lorawan-frequency-plans/master", + "URL for fetching frequency plans") } diff --git a/cmd/version.go b/cmd/version.go index 357ac2d..103239e 100644 --- a/cmd/version.go +++ b/cmd/version.go @@ -1,4 +1,4 @@ -// Copyright © 2020 The Things Network Foundation, The Things Industries B.V. +// Copyright © 2023 The Things Network Foundation, The Things Industries B.V. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -26,25 +26,23 @@ func printVar(k, v string) { fmt.Printf("%-20s %s\n", k+":", v) } -var ( - versionCmd = &cobra.Command{ - Use: "version", - Short: "Print version information", - RunE: func(cmd *cobra.Command, args []string) error { - fmt.Printf("%s: %s\n", cmd.Root().Short, cmd.Root().Name()) - printVar("Version", version.Version) - if version.BuildDate != "" { - printVar("Build date", version.BuildDate) - } - if version.GitCommit != "" { - printVar("Git commit", version.GitCommit) - } - printVar("Go version", runtime.Version()) - printVar("OS/Arch", runtime.GOOS+"/"+runtime.GOARCH) - return nil - }, - } -) +var versionCmd = &cobra.Command{ + Use: "version", + Short: "Print version information", + RunE: func(cmd *cobra.Command, args []string) error { + fmt.Printf("%s: %s\n", cmd.Root().Short, cmd.Root().Name()) + printVar("Version", version.Version) + if version.BuildDate != "" { + printVar("Build date", version.BuildDate) + } + if version.GitCommit != "" { + printVar("Git commit", version.GitCommit) + } + printVar("Go version", runtime.Version()) + printVar("OS/Arch", runtime.GOOS+"/"+runtime.GOARCH) + return nil + }, +} func init() { rootCmd.AddCommand(versionCmd) diff --git a/pkg/source/chirpstack/chirpstack.go b/pkg/source/chirpstack/chirpstack.go index d8dea8c..82e3575 100644 --- a/pkg/source/chirpstack/chirpstack.go +++ b/pkg/source/chirpstack/chirpstack.go @@ -1,4 +1,4 @@ -// Copyright © 2020 The Things Network Foundation, The Things Industries B.V. +// Copyright © 2023 The Things Network Foundation, The Things Industries B.V. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,13 +14,18 @@ package chirpstack -import "go.thethings.network/lorawan-stack-migrate/pkg/source" +import ( + "go.thethings.network/lorawan-stack-migrate/pkg/source" + "go.thethings.network/lorawan-stack-migrate/pkg/source/chirpstack/config" +) func init() { + cfg, flags := config.New() + source.RegisterSource(source.Registration{ Name: "chirpstack", Description: "Migrate from ChirpStack LoRaWAN Network Server", - FlagSet: flagSet(), - Create: NewSource, + FlagSet: flags, + Create: createNewSource(cfg), }) } diff --git a/pkg/source/chirpstack/config.go b/pkg/source/chirpstack/config.go deleted file mode 100644 index 0dc2072..0000000 --- a/pkg/source/chirpstack/config.go +++ /dev/null @@ -1,114 +0,0 @@ -// Copyright © 2020 The Things Network Foundation, The Things Industries B.V. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package chirpstack - -import ( - "context" - "crypto/tls" - "crypto/x509" - "io/ioutil" - "os" - - "github.com/spf13/pflag" - "go.thethings.network/lorawan-stack/v3/pkg/types" -) - -type config struct { - ctx context.Context - - token string - url string - ca string - insecure bool - tls *tls.Config - - frequencyPlanID string - joinEUI *types.EUI64 - exportVars bool - exportSession bool -} - -func flagSet() *pflag.FlagSet { - flags := &pflag.FlagSet{} - flags.String("chirpstack.api-url", os.Getenv("CHIRPSTACK_API_URL"), "ChirpStack API URL") - flags.String("chirpstack.api-token", os.Getenv("CHIRPSTACK_API_TOKEN"), "ChirpStack API Token") - flags.String("chirpstack.api-ca", os.Getenv("CHIRPSTACK_API_CA"), "(optional) CA for TLS") - flags.Bool("chirpstack.api-insecure", os.Getenv("CHIRPSTACK_API_INSECURE") == "1", "Do not connect to ChirpStack over TLS") - flags.Bool("chirpstack.export-vars", false, "Export device variables from ChirpStack") - flags.Bool("chirpstack.export-session", true, "Export device session keys from ChirpStack") - flags.String("chirpstack.join-eui", os.Getenv("JOIN_EUI"), "JoinEUI of exported devices") - flags.String("chirpstack.frequency-plan-id", os.Getenv("FREQUENCY_PLAN_ID"), "Frequency Plan ID of exported devices") - return flags -} - -func buildConfig(ctx context.Context, flags *pflag.FlagSet) (config, error) { - stringFlag := func(f string) string { - s, _ := flags.GetString(f) - return s - } - boolFlag := func(f string) bool { - s, _ := flags.GetBool(f) - return s - } - - c := config{ - ctx: ctx, - - token: stringFlag("chirpstack.api-token"), - url: stringFlag("chirpstack.api-url"), - ca: stringFlag("chirpstack.api-ca"), - insecure: boolFlag("chirpstack.api-insecure"), - - frequencyPlanID: stringFlag("chirpstack.frequency-plan-id"), - exportVars: boolFlag("chirpstack.export-vars"), - exportSession: boolFlag("chirpstack.export-session"), - } - - if c.token == "" { - return config{}, errNoAPIToken.New() - } - if c.url == "" { - return config{}, errNoAPIURL.New() - } - if c.frequencyPlanID == "" { - return config{}, errNoFrequencyPlan.New() - } - - c.joinEUI = &types.EUI64{} - strJoinEUI := stringFlag("chirpstack.join-eui") - if err := c.joinEUI.UnmarshalText([]byte(strJoinEUI)); err != nil { - return config{}, errInvalidJoinEUI.WithAttributes("join_eui", strJoinEUI) - } - - if !c.insecure || c.ca != "" { - c.tls = &tls.Config{} - rootCAs := c.tls.RootCAs - if rootCAs == nil { - var err error - if rootCAs, err = x509.SystemCertPool(); err != nil { - rootCAs = x509.NewCertPool() - } - } - if c.ca != "" { - pemBytes, err := ioutil.ReadFile(c.ca) - if err != nil { - return config{}, errRead.WithAttributes("file", c.ca) - } - rootCAs.AppendCertsFromPEM(pemBytes) - } - } - - return c, nil -} diff --git a/pkg/source/chirpstack/config/config.go b/pkg/source/chirpstack/config/config.go new file mode 100644 index 0000000..0e53784 --- /dev/null +++ b/pkg/source/chirpstack/config/config.go @@ -0,0 +1,151 @@ +// Copyright © 2023 The Things Network Foundation, The Things Industries B.V. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "crypto/tls" + "crypto/x509" + "net/http" + "os" + + "github.com/spf13/pflag" + "go.thethings.network/lorawan-stack-migrate/pkg/source" + "go.thethings.network/lorawan-stack/v3/pkg/types" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" +) + +func New() (*Config, *pflag.FlagSet) { + var ( + config = &Config{} + flags = &pflag.FlagSet{} + ) + + flags.StringVar(&config.url, + "api-url", + os.Getenv("CHIRPSTACK_API_URL"), + "ChirpStack API URL") + flags.StringVar(&config.token, + "api-token", + os.Getenv("CHIRPSTACK_API_TOKEN"), + "ChirpStack API Token") + flags.StringVar(&config.caPath, + "api-ca", + os.Getenv("CHIRPSTACK_API_CA"), + "(optional) CA for TLS") + flags.BoolVar(&config.insecure, + "api-insecure", + os.Getenv("CHIRPSTACK_API_INSECURE") == "1", + "Do not connect to ChirpStack over TLS") + flags.BoolVar(&config.ExportVars, + "export-vars", + false, + "Export device variables from ChirpStack") + flags.BoolVar(&config.ExportSession, + "export-session", + true, + "Export device session keys from ChirpStack") + flags.StringVar(&config.joinEUI, + "join-eui", + os.Getenv("JOIN_EUI"), + "JoinEUI of exported devices") + flags.StringVar(&config.FrequencyPlanID, + "frequency-plan-id", + os.Getenv("FREQUENCY_PLAN_ID"), + "Frequency Plan ID of exported devices") + + return config, flags +} + +type Config struct { + source.Config + + ClientConn *grpc.ClientConn + + token, caPath, url, + FrequencyPlanID string + + joinEUI string + JoinEUI *types.EUI64 + + insecure, + ExportVars, + ExportSession bool +} + +func (c *Config) Initialize() error { + if c.token == "" { + return errNoAPIToken.New() + } + if c.url == "" { + return errNoAPIURL.New() + } + if c.FrequencyPlanID == "" { + return errNoFrequencyPlan.New() + } + + c.JoinEUI = &types.EUI64{} + if err := c.JoinEUI.UnmarshalText([]byte(c.joinEUI)); err != nil { + return errInvalidJoinEUI.WithAttributes("join_eui", c.joinEUI) + } + + if !c.insecure || c.caPath != "" { + if err := setCustomCA(c.caPath); err != nil { + return err + } + } + + err := c.dialGRPC( + grpc.FailOnNonTempDialError(true), + grpc.WithBlock(), + grpc.WithPerRPCCredentials(token(c.token)), + ) + if err != nil { + return err + } + + return nil +} + +func (c *Config) dialGRPC(opts ...grpc.DialOption) error { + if c.insecure && c.caPath == "" { + opts = append(opts, grpc.WithInsecure()) + } + if tls := http.DefaultTransport.(*http.Transport).TLSClientConfig; tls != nil { + opts = append(opts, grpc.WithTransportCredentials(credentials.NewTLS(tls))) + } + var err error + c.ClientConn, err = grpc.Dial(c.url, opts...) + if err != nil { + return err + } + return nil +} + +func setCustomCA(path string) error { + pemBytes, err := os.ReadFile(path) + if err != nil { + return err + } + rootCAs := http.DefaultTransport.(*http.Transport).TLSClientConfig.RootCAs + if rootCAs == nil { + if rootCAs, err = x509.SystemCertPool(); err != nil { + rootCAs = x509.NewCertPool() + } + } + rootCAs.AppendCertsFromPEM(pemBytes) + http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{RootCAs: rootCAs} + return nil +} diff --git a/pkg/source/chirpstack/config/errors.go b/pkg/source/chirpstack/config/errors.go new file mode 100644 index 0000000..9373899 --- /dev/null +++ b/pkg/source/chirpstack/config/errors.go @@ -0,0 +1,25 @@ +// Copyright © 2023 The Things Network Foundation, The Things Industries B.V. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import "go.thethings.network/lorawan-stack/v3/pkg/errors" + +var ( + errNoAPIToken = errors.DefineInvalidArgument("no_api_token", "no API token") + errNoAPIURL = errors.DefineInvalidArgument("no_api_url", "no API URL") + errNoFrequencyPlan = errors.DefineInvalidArgument("no_frequency_plan", "no Frequency Plan") + + errInvalidJoinEUI = errors.DefineInvalidArgument("invalid_join_eui", "invalid JoinEUI `{join_eui}`") +) diff --git a/pkg/source/chirpstack/log.go b/pkg/source/chirpstack/config/log.go similarity index 73% rename from pkg/source/chirpstack/log.go rename to pkg/source/chirpstack/config/log.go index 3493a61..2f10270 100644 --- a/pkg/source/chirpstack/log.go +++ b/pkg/source/chirpstack/config/log.go @@ -1,4 +1,4 @@ -// Copyright © 2020 The Things Network Foundation, The Things Industries B.V. +// Copyright © 2023 The Things Network Foundation, The Things Industries B.V. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,15 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -package chirpstack +package config import "go.thethings.network/lorawan-stack/v3/pkg/log" -func (p *Source) logFields() log.Fielder { +func (c *Config) LogFields() log.Fielder { return log.Fields( - "export_vars", p.exportVars, - "export_session", p.exportSession, - "insecure", p.insecure, - "url", p.url, + "export_vars", c.ExportVars, + "export_session", c.ExportSession, + "insecure", c.insecure, + "url", c.url, ) } diff --git a/pkg/source/chirpstack/token.go b/pkg/source/chirpstack/config/token.go similarity index 92% rename from pkg/source/chirpstack/token.go rename to pkg/source/chirpstack/config/token.go index 51a152d..f8d2eb4 100644 --- a/pkg/source/chirpstack/token.go +++ b/pkg/source/chirpstack/config/token.go @@ -1,4 +1,4 @@ -// Copyright © 2020 The Things Network Foundation, The Things Industries B.V. +// Copyright © 2023 The Things Network Foundation, The Things Industries B.V. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package chirpstack +package config import ( "context" diff --git a/pkg/source/chirpstack/errors.go b/pkg/source/chirpstack/errors.go index 4ca8d45..694f8ac 100644 --- a/pkg/source/chirpstack/errors.go +++ b/pkg/source/chirpstack/errors.go @@ -1,4 +1,4 @@ -// Copyright © 2020 The Things Network Foundation, The Things Industries B.V. +// Copyright © 2023 The Things Network Foundation, The Things Industries B.V. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/pkg/source/chirpstack/source.go b/pkg/source/chirpstack/source.go index b789467..14f2c03 100644 --- a/pkg/source/chirpstack/source.go +++ b/pkg/source/chirpstack/source.go @@ -1,4 +1,4 @@ -// Copyright © 2020 The Things Network Foundation, The Things Industries B.V. +// Copyright © 2023 The Things Network Foundation, The Things Industries B.V. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -23,13 +23,12 @@ import ( csapi "github.com/brocaar/chirpstack-api/go/v3/as/external/api" pbtypes "github.com/gogo/protobuf/types" - "github.com/spf13/pflag" "go.thethings.network/lorawan-stack-migrate/pkg/source" + "go.thethings.network/lorawan-stack-migrate/pkg/source/chirpstack/config" "go.thethings.network/lorawan-stack/v3/pkg/log" "go.thethings.network/lorawan-stack/v3/pkg/ttnpb" "go.thethings.network/lorawan-stack/v3/pkg/types" "google.golang.org/grpc" - "google.golang.org/grpc/credentials" ) const ( @@ -47,47 +46,34 @@ function Decoder(bytes, fport) { // Source implements the Source interface. type Source struct { - config + *config.Config - cc *grpc.ClientConn + ctx context.Context + ClientConn *grpc.ClientConn applications map[int64]*csapi.Application devProfiles map[string]*csapi.DeviceProfile svcProfiles map[string]*csapi.ServiceProfile } -// NewSource creates a new ChirpStack Source. -func NewSource(ctx context.Context, flags *pflag.FlagSet) (source.Source, error) { - p := &Source{} - - var err error - p.config, err = buildConfig(ctx, flags) - if err != nil { - return nil, err - } +func createNewSource(cfg *config.Config) source.CreateSource { + return func(ctx context.Context, _ source.Config) (source.Source, error) { + s := &Source{ + ctx: ctx, + Config: cfg, + } - dialOpts := []grpc.DialOption{ - grpc.FailOnNonTempDialError(true), - grpc.WithBlock(), - grpc.WithPerRPCCredentials(token(p.token)), - } - if p.insecure && p.ca == "" { - dialOpts = append(dialOpts, grpc.WithInsecure()) - } - if p.tls != nil { - dialOpts = append(dialOpts, grpc.WithTransportCredentials(credentials.NewTLS(p.tls))) - } - p.cc, err = grpc.Dial(p.url, dialOpts...) - if err != nil { - return nil, err - } + if err := cfg.Initialize(); err != nil { + return nil, err + } + log.FromContext(s.ctx).WithFields(s.LogFields()).Info("Initialized ChirpStack source") - log.FromContext(p.ctx).WithFields(p.logFields()).Info("Initialized ChirpStack source") + s.applications = make(map[int64]*csapi.Application) + s.devProfiles = make(map[string]*csapi.DeviceProfile) + s.svcProfiles = make(map[string]*csapi.ServiceProfile) - p.applications = make(map[int64]*csapi.Application) - p.devProfiles = make(map[string]*csapi.DeviceProfile) - p.svcProfiles = make(map[string]*csapi.ServiceProfile) - return p, nil + return s, nil + } } // RangeDevices implements the Source interface. @@ -96,7 +82,7 @@ func (p *Source) RangeDevices(id string, f func(source.Source, string) error) er if err != nil { return err } - client := csapi.NewDeviceServiceClient(p.cc) + client := csapi.NewDeviceServiceClient(p.ClientConn) offset := int64(0) for { devices, err := client.List(p.ctx, &csapi.ListDeviceRequest{ @@ -152,7 +138,7 @@ func (p *Source) ExportDevice(devEui string) (*ttnpb.EndDevice, error) { if err != nil { return nil, errInvalidDevEUI.WithAttributes("dev_eui", devEui).WithCause(err) } - dev.Ids.JoinEui = p.joinEUI.Bytes() + dev.Ids.JoinEui = p.JoinEUI.Bytes() dev.Ids.ApplicationIds.ApplicationId = fmt.Sprintf("chirpstack-%d", csdev.ApplicationId) dev.Ids.DeviceId = "eui-" + strings.ToLower(devEui) @@ -166,7 +152,7 @@ func (p *Source) ExportDevice(devEui string) (*ttnpb.EndDevice, error) { for key, value := range csdev.Tags { dev.Attributes[key] = value } - if p.exportVars { + if p.ExportVars { for key, value := range csdev.Variables { dev.Attributes["var-"+key] = value } @@ -181,7 +167,7 @@ func (p *Source) ExportDevice(devEui string) (*ttnpb.EndDevice, error) { } // Frequency Plan - dev.FrequencyPlanId = p.frequencyPlanID + dev.FrequencyPlanId = p.FrequencyPlanID // General switch devProfile.MacVersion { @@ -326,7 +312,7 @@ func (p *Source) ExportDevice(devEui string) (*ttnpb.EndDevice, error) { } // Session - if p.exportSession { + if p.ExportSession { activation, err := p.getActivation(devEui) if err == nil { dev.Session = &ttnpb.Session{Keys: &ttnpb.SessionKeys{}, StartedAt: pbtypes.TimestampNow()} @@ -380,5 +366,5 @@ func (p *Source) ExportDevice(devEui string) (*ttnpb.EndDevice, error) { // Close implements the Source interface. func (p *Source) Close() error { - return p.cc.Close() + return p.ClientConn.Close() } diff --git a/pkg/source/chirpstack/util.go b/pkg/source/chirpstack/util.go index 63347b7..0a43f71 100644 --- a/pkg/source/chirpstack/util.go +++ b/pkg/source/chirpstack/util.go @@ -1,4 +1,4 @@ -// Copyright © 2020 The Things Network Foundation, The Things Industries B.V. +// Copyright © 2023 The Things Network Foundation, The Things Industries B.V. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -36,7 +36,7 @@ func (p *Source) getDeviceProfile(id string) (*csapi.DeviceProfile, error) { return profile, nil } - client := csapi.NewDeviceProfileServiceClient(p.cc) + client := csapi.NewDeviceProfileServiceClient(p.ClientConn) resp, err := client.Get(p.ctx, &csapi.GetDeviceProfileRequest{ Id: id, }) @@ -52,7 +52,7 @@ func (p *Source) getServiceProfile(id string) (*csapi.ServiceProfile, error) { return profile, nil } - client := csapi.NewServiceProfileServiceClient(p.cc) + client := csapi.NewServiceProfileServiceClient(p.ClientConn) resp, err := client.Get(p.ctx, &csapi.GetServiceProfileRequest{ Id: id, }) @@ -64,7 +64,7 @@ func (p *Source) getServiceProfile(id string) (*csapi.ServiceProfile, error) { } func (p *Source) getDevice(devEui string) (*csapi.Device, error) { - client := csapi.NewDeviceServiceClient(p.cc) + client := csapi.NewDeviceServiceClient(p.ClientConn) resp, err := client.Get(p.ctx, &csapi.GetDeviceRequest{ DevEui: devEui, @@ -104,7 +104,7 @@ func (p *Source) getApplicationByID(id int64) (*csapi.Application, error) { return app, nil } - client := csapi.NewApplicationServiceClient(p.cc) + client := csapi.NewApplicationServiceClient(p.ClientConn) resp, err := client.Get(p.ctx, &csapi.GetApplicationRequest{ Id: id, }) @@ -117,7 +117,7 @@ func (p *Source) getApplicationByID(id int64) (*csapi.Application, error) { } func (p *Source) getApplicationIDByName(name string) (int64, error) { - client := csapi.NewApplicationServiceClient(p.cc) + client := csapi.NewApplicationServiceClient(p.ClientConn) offset := int64(0) for { resp, err := client.List(p.ctx, &csapi.ListApplicationRequest{ @@ -141,7 +141,7 @@ func (p *Source) getApplicationIDByName(name string) (int64, error) { } func (p *Source) getRootKeys(devEui string) (*csapi.DeviceKeys, error) { - client := csapi.NewDeviceServiceClient(p.cc) + client := csapi.NewDeviceServiceClient(p.ClientConn) resp, err := client.GetKeys(context.Background(), &csapi.GetDeviceKeysRequest{ DevEui: devEui, }) @@ -153,7 +153,7 @@ func (p *Source) getRootKeys(devEui string) (*csapi.DeviceKeys, error) { } func (p *Source) getActivation(devEui string) (*csapi.DeviceActivation, error) { - client := csapi.NewDeviceServiceClient(p.cc) + client := csapi.NewDeviceServiceClient(p.ClientConn) resp, err := client.GetActivation(context.Background(), &csapi.GetDeviceActivationRequest{ DevEui: devEui, }) diff --git a/pkg/source/source.go b/pkg/source/source.go index e8c772a..fd4e530 100644 --- a/pkg/source/source.go +++ b/pkg/source/source.go @@ -24,6 +24,14 @@ import ( "go.thethings.network/lorawan-stack/v3/pkg/ttnpb" ) +type Config struct { + DryRun, Verbose bool + FrequencyPlansURL, + Source string +} + +var RootConfig Config + // Source is a source for end devices. type Source interface { // ExportDevice retrieves an end device from the source and returns it as a ttnpb.EndDevice. @@ -35,7 +43,7 @@ type Source interface { } // CreateSource is a function that constructs a new Source. -type CreateSource func(ctx context.Context, flags *pflag.FlagSet) (Source, error) +type CreateSource func(ctx context.Context, rootCfg Config) (Source, error) // Registration contains information for a registered Source. type Registration struct { @@ -64,15 +72,25 @@ func RegisterSource(r Registration) error { } // NewSource creates a new Source from parsed flags. -func NewSource(ctx context.Context, flags *pflag.FlagSet) (Source, error) { - sourceName, _ := flags.GetString("source") - if sourceName == "" { +func NewSource(ctx context.Context) (Source, error) { + if RootConfig.Source == "" { return nil, errNoSource.New() } - if registration, ok := registeredSources[sourceName]; ok { - return registration.Create(ctx, flags) + if registration, ok := registeredSources[RootConfig.Source]; ok { + return registration.Create(ctx, RootConfig) } - return nil, errNotRegistered.WithAttributes("source", sourceName) + return nil, errNotRegistered.WithAttributes("source", RootConfig.Source) +} + +func addPrefix(name, prefix string) string { + if prefix == "" { + return name + } + // Append separator + prefix += "." + // Remove prefix if already present + name = strings.TrimPrefix(name, prefix) + return prefix + name } // FlagSet returns flags for all configured sources. @@ -81,11 +99,17 @@ func FlagSet() *pflag.FlagSet { names := []string{} for _, r := range registeredSources { if r.FlagSet != nil { + r.FlagSet.VisitAll(func(f *pflag.Flag) { + f.Name = addPrefix(f.Name, r.Name) + }) flags.AddFlagSet(r.FlagSet) names = append(names, r.Name) } } - flags.String("source", "", fmt.Sprintf("source (%s)", strings.Join(names, "|"))) + flags.StringVar(&RootConfig.Source, + "source", + "", + fmt.Sprintf("source (%s)", strings.Join(Names(), "|"))) return flags } @@ -98,6 +122,15 @@ func Sources() map[string]string { return sources } +// Names returns a slice of registered Sources names. +func Names() []string { + var names []string + for k := range registeredSources { + names = append(names, k) + } + return names +} + func init() { registeredSources = make(map[string]Registration) } diff --git a/pkg/source/ttnv2/config.go b/pkg/source/ttnv2/config.go index 245d545..53b9a36 100644 --- a/pkg/source/ttnv2/config.go +++ b/pkg/source/ttnv2/config.go @@ -1,4 +1,4 @@ -// Copyright © 2020 The Things Network Foundation, The Things Industries B.V. +// Copyright © 2023 The Things Network Foundation, The Things Industries B.V. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -26,6 +26,7 @@ import ( ttnapex "github.com/TheThingsNetwork/go-utils/log/apex" apex "github.com/apex/log" "github.com/spf13/pflag" + "go.thethings.network/lorawan-stack-migrate/pkg/source" "go.thethings.network/lorawan-stack/v3/pkg/fetch" "go.thethings.network/lorawan-stack/v3/pkg/frequencyplans" ) @@ -34,9 +35,68 @@ const ( clientName = "ttn-lw-migrate" ) -type config struct { +func New() (*Config, *pflag.FlagSet) { + var ( + config = &Config{sdkConfig: ttnsdk.NewCommunityConfig(clientName)} + flags = &pflag.FlagSet{} + ) + + flags.StringVar(&config.frequencyPlanID, + "frequency-plan-id", + os.Getenv("FREQUENCY_PLAN_ID"), + "Frequency Plan ID of exported devices") + flags.StringVar(&config.appID, + "app-id", + os.Getenv("TTNV2_APP_ID"), + "TTN Application ID") + flags.StringVar(&config.appAccessKey, + "app-access-key", + os.Getenv("TTNV2_APP_ACCESS_KEY"), + "TTN Application Access Key (with 'devices' permissions") + flags.StringVar(&config.caCert, + "ca-cert", + os.Getenv("TTNV2_CA_CERT"), + "(only for private networks)") + flags.StringVar(&config.sdkConfig.HandlerAddress, + "handler-address", + os.Getenv("TTNV2_HANDLER_ADDRESS"), + "(only for private networks) Address for the Handler") + flags.StringVar(&config.sdkConfig.AccountServerAddress, + "account-server-address", + os.Getenv("TTNV2_ACCOUNT_SERVER_ADDRESS"), + "(only for private networks) Address for the Account Server") + flags.StringVar(&config.sdkConfig.AccountServerClientID, + "account-server-client-id", + os.Getenv("TTNV2_ACCOUNT_SERVER_CLIENT_ID"), + "(only for private networks) Client ID for the Account Server") + flags.StringVar(&config.sdkConfig.AccountServerClientSecret, + "account-server-client-secret", + os.Getenv("TTNV2_ACCOUNT_SERVER_CLIENT_SECRET"), + "(only for private networks) Client secret for the Account Server") + flags.StringVar(&config.sdkConfig.DiscoveryServerAddress, + "discovery-server-address", + os.Getenv("TTNV2_DISCOVERY_SERVER_ADDRESS"), + "(only for private networks) Address for the Discovery Server") + flags.BoolVar(&config.sdkConfig.DiscoveryServerInsecure, + "discovery-server-insecure", + false, + "(only for private networks) Not recommended") + flags.BoolVar(&config.withSession, + "with-session", + true, + "Export device session keys and frame counters") + flags.BoolVar(&config.resetsToFrequencyPlan, + "resets-to-frequency-plan", + false, + "Configure preset frequencies for ABP devices so that they match the used Frequency Plan") + + return config, flags +} + +type Config struct { sdkConfig ttnsdk.ClientConfig + caCert string appAccessKey string appID string @@ -49,81 +109,34 @@ type config struct { fpStore *frequencyplans.Store } -func flagSet() *pflag.FlagSet { - flags := &pflag.FlagSet{} - flags.String("ttnv2.frequency-plan-id", os.Getenv("FREQUENCY_PLAN_ID"), "Frequency Plan ID of exported devices") - flags.String("ttnv2.app-id", os.Getenv("TTNV2_APP_ID"), "TTN Application ID") - flags.String("ttnv2.app-access-key", os.Getenv("TTNV2_APP_ACCESS_KEY"), "TTN Application Access Key (with 'devices' permissions)") - flags.String("ttnv2.ca-cert", os.Getenv("TTNV2_CA_CERT"), "(only for private networks) CA for TLS") - flags.String("ttnv2.handler-address", os.Getenv("TTNV2_HANDLER_ADDRESS"), "(only for private networks) Address for the Handler") - flags.String("ttnv2.account-server-address", os.Getenv("TTNV2_ACCOUNT_SERVER_ADDRESS"), "(only for private networks) Address for the Account Server") - flags.String("ttnv2.account-server-client-id", os.Getenv("TTNV2_ACCOUNT_SERVER_CLIENT_ID"), "(only for private networks) Client ID for the Account Server") - flags.String("ttnv2.account-server-client-secret", os.Getenv("TTNV2_ACCOUNT_SERVER_CLIENT_SECRET"), "(only for private networks) Client secret for the Account Server") - flags.String("ttnv2.discovery-server-address", os.Getenv("TTNV2_DISCOVERY_SERVER_ADDRESS"), "(only for private networks) Address for the Discovery Server") - flags.Bool("ttnv2.discovery-server-insecure", false, "(only for private networks) Not recommended") - flags.Bool("ttnv2.with-session", true, "Export device session keys and frame counters") - flags.Bool("ttnv2.resets-to-frequency-plan", false, "Configure preset frequencies for ABP devices so that they match the used Frequency Plan") - - return flags -} - -func getConfig(flags *pflag.FlagSet) (config, error) { - stringFlag := func(f string) string { - s, _ := flags.GetString(f) - return s - } - boolFlag := func(f string) bool { - s, _ := flags.GetBool(f) - return s - } - - cfg := ttnsdk.NewCommunityConfig(clientName) - if f := stringFlag("ttnv2.account-server-address"); f != "" { - cfg.AccountServerAddress = f - } - if f := stringFlag("ttnv2.account-server-client-id"); f != "" { - cfg.AccountServerClientID = f - } - if f := stringFlag("ttnv2.account-server-client-secret"); f != "" { - cfg.AccountServerClientSecret = f - } - if f := stringFlag("ttnv2.handler-address"); f != "" { - cfg.HandlerAddress = f - } - if f := stringFlag("ttnv2.discovery-server-address"); f != "" { - cfg.DiscoveryServerAddress = f - } - cfg.DiscoveryServerInsecure = boolFlag("ttnv2.discovery-server-insecure") - - if ca := stringFlag("ttnv2.ca-cert"); ca != "" { - if cfg.TLSConfig == nil { - cfg.TLSConfig = &tls.Config{} +func (c *Config) Initialize(rootConfig source.Config) error { + if c.caCert != "" { + if c.sdkConfig.TLSConfig == nil { + c.sdkConfig.TLSConfig = new(tls.Config) } - rootCAs := cfg.TLSConfig.RootCAs + rootCAs := c.sdkConfig.TLSConfig.RootCAs if rootCAs == nil { var err error if rootCAs, err = x509.SystemCertPool(); err != nil { rootCAs = x509.NewCertPool() } } - pemBytes, err := ioutil.ReadFile(ca) + pemBytes, err := ioutil.ReadFile(c.caCert) if err != nil { - return config{}, errRead.WithAttributes("file", ca) + return err } rootCAs.AppendCertsFromPEM(pemBytes) } - appAccessKey := stringFlag("ttnv2.app-access-key") - if appAccessKey == "" { - return config{}, errNoAppAccessKey.New() + if c.appAccessKey == "" { + return errNoAppAccessKey.New() } - frequencyPlanID := stringFlag("ttnv2.frequency-plan-id") - if frequencyPlanID == "" { - return config{}, errNoFrequencyPlanID.New() + if c.frequencyPlanID == "" { + return errNoFrequencyPlanID.New() } logLevel := ttnapex.InfoLevel - if boolFlag("verbose") { + if rootConfig.Verbose { logLevel = ttnapex.DebugLevel } logger := ttnapex.Wrap(&apex.Logger{ @@ -132,22 +145,13 @@ func getConfig(flags *pflag.FlagSet) (config, error) { }) ttnlog.Set(logger) - fpFetcher, err := fetch.FromHTTP(nil, stringFlag("frequency-plans-url")) + fpFetcher, err := fetch.FromHTTP(nil, rootConfig.FrequencyPlansURL) if err != nil { - return config{}, err + return err } + c.fpStore = frequencyplans.NewStore(fpFetcher) - return config{ - sdkConfig: cfg, - - appID: stringFlag("ttnv2.app-id"), - appAccessKey: appAccessKey, - frequencyPlanID: frequencyPlanID, - - withSession: boolFlag("ttnv2.with-session"), - resetsToFrequencyPlan: boolFlag("ttnv2.resets-to-frequency-plan"), + c.dryRun = rootConfig.DryRun - dryRun: boolFlag("dry-run"), - fpStore: frequencyplans.NewStore(fpFetcher), - }, nil + return nil } diff --git a/pkg/source/ttnv2/errors.go b/pkg/source/ttnv2/errors.go index 1aa62be..15565f6 100644 --- a/pkg/source/ttnv2/errors.go +++ b/pkg/source/ttnv2/errors.go @@ -1,4 +1,4 @@ -// Copyright © 2020 The Things Network Foundation, The Things Industries B.V. +// Copyright © 2023 The Things Network Foundation, The Things Industries B.V. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/pkg/source/ttnv2/retry.go b/pkg/source/ttnv2/retry.go index c782d8d..0d3e6fb 100644 --- a/pkg/source/ttnv2/retry.go +++ b/pkg/source/ttnv2/retry.go @@ -1,4 +1,4 @@ -// Copyright © 2021 The Things Network Foundation, The Things Industries B.V. +// Copyright © 2023 The Things Network Foundation, The Things Industries B.V. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/pkg/source/ttnv2/source.go b/pkg/source/ttnv2/source.go index 831c4e5..1f4ba79 100644 --- a/pkg/source/ttnv2/source.go +++ b/pkg/source/ttnv2/source.go @@ -20,7 +20,6 @@ import ( ttnsdk "github.com/TheThingsNetwork/go-app-sdk" ttntypes "github.com/TheThingsNetwork/ttn/core/types" pbtypes "github.com/gogo/protobuf/types" - "github.com/spf13/pflag" "go.thethings.network/lorawan-stack-migrate/pkg/source" "go.thethings.network/lorawan-stack/v3/pkg/errors" "go.thethings.network/lorawan-stack/v3/pkg/log" @@ -33,29 +32,30 @@ import ( type Source struct { ctx context.Context - config config + config *Config mgr ttnsdk.DeviceManager client ttnsdk.Client } -// NewSource creates a new TTNv2 Source. -func NewSource(ctx context.Context, flags *pflag.FlagSet) (source.Source, error) { - config, err := getConfig(flags) - if err != nil { - return nil, err +func createNewSource(cfg *Config) source.CreateSource { + return func(ctx context.Context, rootCfg source.Config) (source.Source, error) { + return NewSource(ctx, cfg, rootCfg) } +} +// NewSource creates a new TTNv2 Source. +func NewSource(ctx context.Context, cfg *Config, rootCfg source.Config) (source.Source, error) { s := &Source{ ctx: ctx, - config: config, - client: config.sdkConfig.NewClient(config.appID, config.appAccessKey), + config: cfg, + client: cfg.sdkConfig.NewClient(cfg.appID, cfg.appAccessKey), } mgr, err := s.client.ManageDevices() if err != nil { return nil, err } s.mgr = newDeviceManager(ctx, mgr) - return s, nil + return s, cfg.Initialize(rootCfg) } // ExportDevice implements the source.Source interface. diff --git a/pkg/source/ttnv2/ttnv2.go b/pkg/source/ttnv2/ttnv2.go index 7d7af9a..a5f4093 100644 --- a/pkg/source/ttnv2/ttnv2.go +++ b/pkg/source/ttnv2/ttnv2.go @@ -1,4 +1,4 @@ -// Copyright © 2020 The Things Network Foundation, The Things Industries B.V. +// Copyright © 2023 The Things Network Foundation, The Things Industries B.V. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -17,10 +17,12 @@ package ttnv2 import "go.thethings.network/lorawan-stack-migrate/pkg/source" func init() { + cfg, flags := New() + source.RegisterSource(source.Registration{ Name: "ttnv2", Description: "Migrate from The Things Network Stack V2", - FlagSet: flagSet(), - Create: NewSource, + FlagSet: flags, + Create: createNewSource(cfg), }) } diff --git a/pkg/source/ttnv3/api/api.go b/pkg/source/ttnv3/api/api.go index 50eefe3..6a16068 100644 --- a/pkg/source/ttnv3/api/api.go +++ b/pkg/source/ttnv3/api/api.go @@ -1,3 +1,17 @@ +// Copyright © 2023 The Things Network Foundation, The Things Industries B.V. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package api import ( diff --git a/pkg/source/ttnv3/config.go b/pkg/source/ttnv3/config.go deleted file mode 100644 index 8794b1f..0000000 --- a/pkg/source/ttnv3/config.go +++ /dev/null @@ -1,135 +0,0 @@ -package ttnv3 - -import ( - "crypto/tls" - "crypto/x509" - "fmt" - "net/http" - "os" - - "github.com/spf13/pflag" - "go.uber.org/zap" - - "go.thethings.network/lorawan-stack-migrate/pkg/source/ttnv3/api" -) - -var logger *zap.SugaredLogger - -type config struct { - appID string - - identityServerGRPCAddress, - joinServerGRPCAddress, - applicationServerGRPCAddress, - networkServerGRPCAddress string - - deleteSourceDevice, - dryRun, - noSession bool -} - -func flagSet() *pflag.FlagSet { - flags := &pflag.FlagSet{} - flags.String(flagWithPrefix("app-id"), os.Getenv("TTNV3_APP_ID"), "TTS Application ID") - flags.String(flagWithPrefix("app-api-key"), os.Getenv("TTNV3_APP_API_KEY"), "TTS Application Access Key (with 'devices' permissions)") - flags.String(flagWithPrefix("ca-file"), os.Getenv("TTNV3_CA_FILE"), "TTS Path to a CA file (optional)") - flags.String(flagWithPrefix("identity-server-grpc-address"), os.Getenv("TTNV3_IDENTITY_SERVER_GRPC_ADDRESS"), "TTS Identity Server GRPC Address") - flags.String(flagWithPrefix("join-server-grpc-address"), os.Getenv("TTNV3_JOIN_SERVER_GRPC_ADDRESS"), "TTS Join Server GRPC Address") - flags.String(flagWithPrefix("application-server-grpc-address"), os.Getenv("TTNV3_APPLICATION_SERVER_GRPC_ADDRESS"), "TTS Application Server GRPC Address") - flags.String(flagWithPrefix("network-server-grpc-address"), os.Getenv("TTNV3_NETWORK_SERVER_GRPC_ADDRESS"), "TTS Network Server GRPC Address") - flags.Bool(flagWithPrefix("insecure"), false, "TTS allow TCP connection") - flags.Bool(flagWithPrefix("no-session"), false, "TTS export devices without session") - flags.Bool(flagWithPrefix("delete-source-device"), false, "TTS delete exported devices") - return flags -} - -func getConfig(flags *pflag.FlagSet) (*config, error) { - stringFlag := func(f string) string { - s, _ := flags.GetString(f) - return s - } - boolFlag := func(f string) bool { - b, _ := flags.GetBool(f) - return b - } - - apiKey := stringFlag(flagWithPrefix("app-api-key")) - if apiKey == "" { - return nil, errNoAppAPIKey.New() - } - api.SetAuth("bearer", apiKey) - - http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{} - if insecure := boolFlag(flagWithPrefix("insecure")); insecure { - api.SetInsecure(true) - logger.Warn("Using insecure connection to API") - } else if caPath := stringFlag(flagWithPrefix("ca-file")); caPath != "" { - pemBytes, err := os.ReadFile(caPath) - if err != nil { - return nil, err - } - rootCAs := http.DefaultTransport.(*http.Transport).TLSClientConfig.RootCAs - if rootCAs == nil { - if rootCAs, err = x509.SystemCertPool(); err != nil { - rootCAs = x509.NewCertPool() - } - } - rootCAs.AppendCertsFromPEM(pemBytes) - http.DefaultTransport.(*http.Transport).TLSClientConfig.RootCAs = rootCAs - if err = api.AddCA(pemBytes); err != nil { - return nil, err - } - } - - identityServerGRPCAddress := stringFlag(flagWithPrefix("identity-server-grpc-address")) - if identityServerGRPCAddress == "" { - return nil, errNoIdentityServerGRPCAddress.New() - } - joinServerGRPCAddress := stringFlag(flagWithPrefix("join-server-grpc-address")) - if joinServerGRPCAddress == "" { - return nil, errNoJoinServerGRPCAddress.New() - } - applicationServerGRPCAddress := stringFlag(flagWithPrefix("application-server-grpc-address")) - if applicationServerGRPCAddress == "" { - return nil, errNoApplicationServerGRPCAddress.New() - } - networkServerGRPCAddress := stringFlag(flagWithPrefix("network-server-grpc-address")) - if networkServerGRPCAddress == "" { - return nil, errNoNetworkServerGRPCAddress.New() - } - cfg := zap.NewProductionConfig() - if boolFlag("verbose") { - cfg.Level = zap.NewAtomicLevelAt(zap.DebugLevel) - } - zapLogger, err := cfg.Build() - if err != nil { - return nil, err - } - logger = zapLogger.Sugar() - - dryRun := boolFlag("dry-run") - deleteSourceDevice := boolFlag(flagWithPrefix("delete-source-device")) - // deleteSourceDevice not allowed when dryRun - if dryRun && deleteSourceDevice { - logger.Warn("Cannot delete source device during a dry run.") - deleteSourceDevice = false - } - noSession := boolFlag(flagWithPrefix("no-session")) - - return &config{ - appID: stringFlag(flagWithPrefix("app-id")), - - identityServerGRPCAddress: identityServerGRPCAddress, - joinServerGRPCAddress: joinServerGRPCAddress, - applicationServerGRPCAddress: applicationServerGRPCAddress, - networkServerGRPCAddress: networkServerGRPCAddress, - - deleteSourceDevice: deleteSourceDevice, - dryRun: dryRun, - noSession: noSession, - }, nil -} - -func flagWithPrefix(f string) string { - return fmt.Sprintf("ttnv3.%s", f) -} diff --git a/pkg/source/ttnv3/config/config.go b/pkg/source/ttnv3/config/config.go new file mode 100644 index 0000000..1119592 --- /dev/null +++ b/pkg/source/ttnv3/config/config.go @@ -0,0 +1,219 @@ +// Copyright © 2023 The Things Network Foundation, The Things Industries B.V. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "crypto/tls" + "crypto/x509" + "fmt" + "net/http" + "os" + + "github.com/spf13/pflag" + "go.thethings.network/lorawan-stack-migrate/pkg/source" + "go.thethings.network/lorawan-stack-migrate/pkg/source/ttnv3/api" + "go.uber.org/zap" +) + +var logger *zap.SugaredLogger + +type serverConfig struct { + defaultGRPCAddress, + ApplicationServerGRPCAddress, + IdentityServerGRPCAddress, + JoinServerGRPCAddress, + NetworkServerGRPCAddress string +} + +func (c *serverConfig) anyFieldEmpty() error { + if c.ApplicationServerGRPCAddress == "" { + return errNoApplicationServerGRPCAddress.New() + } + if c.IdentityServerGRPCAddress == "" { + return errNoIdentityServerGRPCAddress.New() + } + if c.JoinServerGRPCAddress == "" { + return errNoJoinServerGRPCAddress.New() + } + if c.NetworkServerGRPCAddress == "" { + return errNoNetworkServerGRPCAddress.New() + } + return nil +} + +func (c *serverConfig) applyDefaults() { + applyDefault := func(adresses ...*string) { + for _, a := range adresses { + if *a == "" { + *a = c.defaultGRPCAddress + } + } + } + + applyDefault( + &c.ApplicationServerGRPCAddress, + &c.IdentityServerGRPCAddress, + &c.JoinServerGRPCAddress, + &c.NetworkServerGRPCAddress, + ) +} + +func New() (*Config, *pflag.FlagSet) { + var ( + config = &Config{ServerConfig: &serverConfig{}} + flags = &pflag.FlagSet{} + ) + + flags.StringVar(&config.AppID, + "app-id", + os.Getenv("TTNV3_APP_ID"), + "TTS Application ID") + flags.StringVar(&config.appAPIKey, + "app-api-key", + os.Getenv("TTNV3_APP_API_KEY"), + "TTS Application Access Key (with 'devices' permissions)") + + flags.StringVar(&config.caPath, + "ca-file", + os.Getenv("TTNV3_CA_FILE"), + "TTS Path to a CA file (optional)") + flags.BoolVar(&config.insecure, + "insecure", + false, + "TTS allow TCP connection") + + flags.StringVar(&config.ServerConfig.defaultGRPCAddress, + "default-grpc-address", + os.Getenv("TTNV3_DEFAULT_GRPC_ADDRESS"), + "TTS default GRPC Address (optional)") + flags.StringVar(&config.ServerConfig.ApplicationServerGRPCAddress, + "appplication-server-grpc-address", + os.Getenv("TTNV3_APPLICATION_SERVER_GRPC_ADDRESS"), + "TTS Application Server GRPC Address") + flags.StringVar(&config.ServerConfig.IdentityServerGRPCAddress, + "identity-server-grpc-address", + os.Getenv("TTNV3_IDENTITY_SERVER_GRPC_ADDRESS"), + "TTS Identity Server GRPC Address") + flags.StringVar(&config.ServerConfig.JoinServerGRPCAddress, + "join-server-grpc-address", + os.Getenv("TTNV3_JOIN_SERVER_GRPC_ADDRESS"), + "TTS Join Server GRPC Address") + flags.StringVar(&config.ServerConfig.NetworkServerGRPCAddress, + "network-server-grpc-address", + os.Getenv("TTNV3_NETWORK_SERVER_GRPC_ADDRESS"), + "TTS Network Server GRPC Address") + + flags.BoolVar(&config.NoSession, + "no-session", + false, + "TTS export devices without session") + flags.BoolVar(&config.DeleteSourceDevice, + "delete-source-device", + false, + "TTS delete exported devices") + + return config, flags +} + +type Config struct { + source.Config + + ServerConfig *serverConfig + + caPath, appAPIKey, + AppID string + + insecure, NoSession, + DeleteSourceDevice bool +} + +func (c *Config) Initialize(rootConfig source.Config) error { + c.Config = rootConfig + + var err error + logger, err = NewLogger(c.Verbose) + if err != nil { + return err + } + + if c.appAPIKey == "" { + return errNoAppAPIKey.New() + } + api.SetAuth("bearer", c.appAPIKey) + + switch { + case c.insecure: + api.SetInsecure(true) + logger.Warn("Using insecure connection to API") + + default: + if c.caPath != "" { + setCustomCA(c.caPath) + } + } + + c.ServerConfig.applyDefaults() + if err := c.ServerConfig.anyFieldEmpty(); err != nil { + return err + } + + // DeleteSourceDevice is not allowed during a dry run + if c.DryRun && c.DeleteSourceDevice { + logger.Warn("Cannot delete source devices during a dry run.") + c.DeleteSourceDevice = false + } + + return nil +} + +func NewLogger(verbose bool) (*zap.SugaredLogger, error) { + cfg := zap.NewProductionConfig() + if verbose { + cfg.Level = zap.NewAtomicLevelAt(zap.DebugLevel) + } + logger, err := cfg.Build() + if err != nil { + return nil, err + } + return logger.Sugar(), nil +} + +func setCustomCA(path string) error { + pemBytes, err := os.ReadFile(path) + if err != nil { + return err + } + + cfg := http.DefaultTransport.(*http.Transport).TLSClientConfig + switch { + case cfg == nil: + cfg = new(tls.Config) + fallthrough + + case cfg.RootCAs == nil: + if cfg.RootCAs, err = x509.SystemCertPool(); err != nil { + cfg.RootCAs = x509.NewCertPool() + } + } + cfg.RootCAs.AppendCertsFromPEM(pemBytes) + if err = api.AddCA(pemBytes); err != nil { + return err + } + return nil +} + +func flagWithPrefix(f string) string { + return fmt.Sprintf("ttnv3.%s", f) +} diff --git a/pkg/source/ttnv3/config/errors.go b/pkg/source/ttnv3/config/errors.go new file mode 100644 index 0000000..43f37ee --- /dev/null +++ b/pkg/source/ttnv3/config/errors.go @@ -0,0 +1,25 @@ +// Copyright © 2023 The Things Network Foundation, The Things Industries B.V. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import "go.thethings.network/lorawan-stack/v3/pkg/errors" + +var ( + errNoAppAPIKey = errors.DefineInvalidArgument("no_app_api_key", "no app api key") + errNoIdentityServerGRPCAddress = errors.DefineInvalidArgument("no_identity_server_grpc_address", "no identity server grpc address") + errNoJoinServerGRPCAddress = errors.DefineInvalidArgument("no_join_server_grpc_address", "no join server grpc address") + errNoApplicationServerGRPCAddress = errors.DefineInvalidArgument("no_application_server_grpc_address", "no application server grpc address") + errNoNetworkServerGRPCAddress = errors.DefineInvalidArgument("no_network_server_grpc_address", "no network server grpc address") +) diff --git a/pkg/source/ttnv3/errors.go b/pkg/source/ttnv3/errors.go index a611600..4a72a3e 100644 --- a/pkg/source/ttnv3/errors.go +++ b/pkg/source/ttnv3/errors.go @@ -1,3 +1,17 @@ +// Copyright © 2023 The Things Network Foundation, The Things Industries B.V. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package ttnv3 import "go.thethings.network/lorawan-stack/v3/pkg/errors" diff --git a/pkg/source/ttnv3/source.go b/pkg/source/ttnv3/source.go index 9e9190c..94c7bed 100644 --- a/pkg/source/ttnv3/source.go +++ b/pkg/source/ttnv3/source.go @@ -1,38 +1,50 @@ +// Copyright © 2023 The Things Network Foundation, The Things Industries B.V. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package ttnv3 import ( "context" - "github.com/spf13/pflag" - "go.thethings.network/lorawan-stack/v3/pkg/ttnpb" - "go.thethings.network/lorawan-stack-migrate/pkg/source" "go.thethings.network/lorawan-stack-migrate/pkg/source/ttnv3/api" + "go.thethings.network/lorawan-stack-migrate/pkg/source/ttnv3/config" + "go.thethings.network/lorawan-stack/v3/pkg/ttnpb" + "go.uber.org/zap" ) +var logger *zap.SugaredLogger + // Source implements the Source interface. type Source struct { ctx context.Context - config *config + config *config.Config } -// NewSource creates a new TTNv3 cource -func NewSource(ctx context.Context, flags *pflag.FlagSet) (source.Source, error) { - config, err := getConfig(flags) - if err != nil { - return Source{}, err - } - s := Source{ - ctx: ctx, - config: config, +func createNewSource(cfg *config.Config) source.CreateSource { + return func(ctx context.Context, rootCfg source.Config) (source.Source, error) { + return Source{ + ctx: ctx, + config: cfg, + }, cfg.Initialize(rootCfg) } - return s, nil } // ExportDevice implements the source.Source interface. func (s Source) ExportDevice(devID string) (*ttnpb.EndDevice, error) { - if s.config.appID == "" { + if s.config.AppID == "" { return nil, errNoAppID.New() } @@ -46,12 +58,12 @@ func (s Source) ExportDevice(devID string) (*ttnpb.EndDevice, error) { if len(jsPaths) > 0 { isPaths = ttnpb.AddFields(isPaths, "join_server_address") } - is, err := api.Dial(s.ctx, s.config.identityServerGRPCAddress) + is, err := api.Dial(s.ctx, s.config.ServerConfig.IdentityServerGRPCAddress) if err != nil { return nil, err } ids := &ttnpb.EndDeviceIdentifiers{ - ApplicationIds: &ttnpb.ApplicationIdentifiers{ApplicationId: s.config.appID}, + ApplicationIds: &ttnpb.ApplicationIdentifiers{ApplicationId: s.config.AppID}, DeviceId: devID, } dev, err := ttnpb.NewEndDeviceRegistryClient(is).Get(s.ctx, &ttnpb.GetEndDeviceRequest{ @@ -81,17 +93,17 @@ func (s Source) ExportDevice(devID string) (*ttnpb.EndDevice, error) { return nil, err } - if s.config.noSession { + if s.config.NoSession { if err := clearDeviceSession(dev); err != nil { return nil, err } } switch { - case s.config.deleteSourceDevice: + case s.config.DeleteSourceDevice: if err := s.deleteEndDevice(dev.GetIds()); err != nil { return nil, err } - case !s.config.dryRun: + case !s.config.DryRun: d := &ttnpb.EndDevice{ Ids: dev.GetIds(), MacSettings: &ttnpb.MACSettings{ScheduleDownlinks: &ttnpb.BoolValue{Value: false}}, @@ -108,9 +120,9 @@ func (s Source) ExportDevice(devID string) (*ttnpb.EndDevice, error) { // RangeDevices implements the source.Source interface. func (s Source) RangeDevices(appID string, f func(source.Source, string) error) error { - s.config.appID = appID + s.config.AppID = appID - is, err := api.Dial(s.ctx, s.config.identityServerGRPCAddress) + is, err := api.Dial(s.ctx, s.config.ServerConfig.IdentityServerGRPCAddress) if err != nil { return err } @@ -147,10 +159,10 @@ func (s Source) Close() error { func (s Source) getEndDevice(ids *ttnpb.EndDeviceIdentifiers, nsPaths, asPaths, jsPaths []string) (*ttnpb.EndDevice, error) { res := &ttnpb.EndDevice{} if len(jsPaths) > 0 { - if s.config.joinServerGRPCAddress == "" { + if s.config.ServerConfig.JoinServerGRPCAddress == "" { logger.With("paths", jsPaths).Warn("Join Server disabled but fields specified to get") } else { - js, err := api.Dial(s.ctx, s.config.joinServerGRPCAddress) + js, err := api.Dial(s.ctx, s.config.ServerConfig.JoinServerGRPCAddress) if err != nil { return nil, err } @@ -172,10 +184,10 @@ func (s Source) getEndDevice(ids *ttnpb.EndDeviceIdentifiers, nsPaths, asPaths, } } if len(asPaths) > 0 { - if s.config.applicationServerGRPCAddress == "" { + if s.config.ServerConfig.ApplicationServerGRPCAddress == "" { logger.With("paths", asPaths).Warn("Application Server disabled but fields specified to get") } else { - as, err := api.Dial(s.ctx, s.config.applicationServerGRPCAddress) + as, err := api.Dial(s.ctx, s.config.ServerConfig.ApplicationServerGRPCAddress) if err != nil { return nil, err } @@ -196,10 +208,10 @@ func (s Source) getEndDevice(ids *ttnpb.EndDeviceIdentifiers, nsPaths, asPaths, } } if len(nsPaths) > 0 { - if s.config.networkServerGRPCAddress == "" { + if s.config.ServerConfig.NetworkServerGRPCAddress == "" { logger.With("paths", nsPaths).Warn("Network Server disabled but fields specified to get") } else { - ns, err := api.Dial(s.ctx, s.config.networkServerGRPCAddress) + ns, err := api.Dial(s.ctx, s.config.ServerConfig.NetworkServerGRPCAddress) if err != nil { return nil, err } @@ -230,7 +242,7 @@ func (s Source) setEndDevice(device *ttnpb.EndDevice, isPaths, nsPaths, asPaths, return nil, err } if len(isPaths) > 0 { - is, err := api.Dial(s.ctx, s.config.identityServerGRPCAddress) + is, err := api.Dial(s.ctx, s.config.ServerConfig.IdentityServerGRPCAddress) if err != nil { return nil, err } @@ -250,10 +262,10 @@ func (s Source) setEndDevice(device *ttnpb.EndDevice, isPaths, nsPaths, asPaths, updateDeviceTimestamps(&res, isRes) } if len(jsPaths) > 0 { - if s.config.joinServerGRPCAddress == "" { + if s.config.ServerConfig.JoinServerGRPCAddress == "" { logger.With("paths", jsPaths).Warn("Join Server disabled but fields specified to set") } else { - js, err := api.Dial(s.ctx, s.config.joinServerGRPCAddress) + js, err := api.Dial(s.ctx, s.config.ServerConfig.JoinServerGRPCAddress) if err != nil { return nil, err } @@ -276,10 +288,10 @@ func (s Source) setEndDevice(device *ttnpb.EndDevice, isPaths, nsPaths, asPaths, } } if len(nsPaths) > 0 { - if s.config.networkServerGRPCAddress == "" { + if s.config.ServerConfig.NetworkServerGRPCAddress == "" { logger.With("paths", nsPaths).Warn("Network Server disabled but fields specified to set") } else { - ns, err := api.Dial(s.ctx, s.config.networkServerGRPCAddress) + ns, err := api.Dial(s.ctx, s.config.ServerConfig.NetworkServerGRPCAddress) if err != nil { return nil, err } @@ -302,10 +314,10 @@ func (s Source) setEndDevice(device *ttnpb.EndDevice, isPaths, nsPaths, asPaths, } } if len(asPaths) > 0 { - if s.config.applicationServerGRPCAddress == "" { + if s.config.ServerConfig.ApplicationServerGRPCAddress == "" { logger.With("paths", asPaths).Warn("Application Server disabled but fields specified to set") } else { - as, err := api.Dial(s.ctx, s.config.applicationServerGRPCAddress) + as, err := api.Dial(s.ctx, s.config.ServerConfig.ApplicationServerGRPCAddress) if err != nil { return nil, err } @@ -328,7 +340,7 @@ func (s Source) setEndDevice(device *ttnpb.EndDevice, isPaths, nsPaths, asPaths, } func (s Source) deleteEndDevice(ids *ttnpb.EndDeviceIdentifiers) error { - if address := s.config.applicationServerGRPCAddress; address != "" { + if address := s.config.ServerConfig.ApplicationServerGRPCAddress; address != "" { as, err := api.Dial(s.ctx, address) if err != nil { return err @@ -337,7 +349,7 @@ func (s Source) deleteEndDevice(ids *ttnpb.EndDeviceIdentifiers) error { return err } } - if address := s.config.networkServerGRPCAddress; address != "" { + if address := s.config.ServerConfig.NetworkServerGRPCAddress; address != "" { ns, err := api.Dial(s.ctx, address) if err != nil { return err @@ -346,7 +358,7 @@ func (s Source) deleteEndDevice(ids *ttnpb.EndDeviceIdentifiers) error { return err } } - if address := s.config.joinServerGRPCAddress; address != "" { + if address := s.config.ServerConfig.JoinServerGRPCAddress; address != "" { js, err := api.Dial(s.ctx, address) if err != nil { return err @@ -355,7 +367,7 @@ func (s Source) deleteEndDevice(ids *ttnpb.EndDeviceIdentifiers) error { return err } } - if address := s.config.identityServerGRPCAddress; address != "" { + if address := s.config.ServerConfig.IdentityServerGRPCAddress; address != "" { is, err := api.Dial(s.ctx, address) if err != nil { return err diff --git a/pkg/source/ttnv3/ttnv3.go b/pkg/source/ttnv3/ttnv3.go index 0714899..1978122 100644 --- a/pkg/source/ttnv3/ttnv3.go +++ b/pkg/source/ttnv3/ttnv3.go @@ -1,14 +1,33 @@ +// Copyright © 2023 The Things Network Foundation, The Things Industries B.V. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package ttnv3 import ( "go.thethings.network/lorawan-stack-migrate/pkg/source" + "go.thethings.network/lorawan-stack-migrate/pkg/source/ttnv3/config" ) func init() { + cfg, flags := config.New() + + logger, _ = config.NewLogger(cfg.Verbose) + source.RegisterSource(source.Registration{ Name: "ttnv3", Description: "Migrate from The Things Stack", - FlagSet: flagSet(), - Create: NewSource, + FlagSet: flags, + Create: createNewSource(cfg), }) } diff --git a/pkg/source/ttnv3/util.go b/pkg/source/ttnv3/util.go index 35963b1..1969023 100644 --- a/pkg/source/ttnv3/util.go +++ b/pkg/source/ttnv3/util.go @@ -1,3 +1,17 @@ +// Copyright © 2023 The Things Network Foundation, The Things Industries B.V. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package ttnv3 import (