Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: New --private-key-file flag when creating a resource instance #670

Merged
merged 5 commits into from Apr 3, 2023
Merged

feat: New --private-key-file flag when creating a resource instance #670

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
6019c46
allow for using a private key file flag for SSH tunnels or Snowflake
Mar 30, 2023
1275ac1
rebuilt docs
Mar 30, 2023
24c319f
lint and docs
Mar 30, 2023
5a460e0
Update cmd/meroxa/root/resources/create.go
janelletavares Mar 30, 2023
a9ae1cd
better example
Mar 30, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
119 changes: 89 additions & 30 deletions cmd/meroxa/root/resources/create.go
View file
Open in desktop
Expand Up @@ -20,6 +20,7 @@ import (
"context"
"encoding/json"
"fmt"
"os"

"github.com/google/uuid"
"github.com/meroxa/cli/cmd/meroxa/builder"
Expand Down Expand Up @@ -49,15 +50,16 @@ type Create struct {
Environment string `long:"env" usage:"environment (name or UUID) where resource will be created"`

// credentials
Username string `long:"username" short:"" usage:"username"`
Password string `long:"password" short:"" usage:"password"`
CaCert string `long:"ca-cert" short:"" usage:"trusted certificates for verifying resource"`
ClientCert string `long:"client-cert" short:"" usage:"client certificate for authenticating to the resource"`
ClientKey string `long:"client-key" short:"" usage:"client private key for authenticating to the resource"`
SSL bool `long:"ssl" short:"" usage:"use SSL"`
SSHURL string `long:"ssh-url" short:"" usage:"SSH tunneling address"`
SSHPrivateKey string `long:"ssh-private-key" short:"" usage:"SSH tunneling private key"`
Token string `long:"token" short:"" usage:"API Token"`
Username string `long:"username" short:"" usage:"username"`
Password string `long:"password" short:"" usage:"password"`
CaCert string `long:"ca-cert" short:"" usage:"trusted certificates for verifying resource"`
ClientCert string `long:"client-cert" short:"" usage:"client certificate for authenticating to the resource"`
ClientKey string `long:"client-key" short:"" usage:"client private key for authenticating to the resource"`
SSL bool `long:"ssl" short:"" usage:"use SSL"`
SSHURL string `long:"ssh-url" short:"" usage:"SSH tunneling address"`
SSHPrivateKey string `long:"ssh-private-key" short:"" usage:"SSH tunneling private key"`
PrivateKeyFile string `long:"private-key-file" short:"" usage:"path to private key file"`
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That's neat and as you already said, super helpful with creating snowflake resources in particular!

Token string `long:"token" short:"" usage:"API Token"`
}
}

Expand All @@ -82,41 +84,72 @@ func (c *Create) Docs() builder.Docs {

// TODO: Provide example with `--env` once it's not behind a feature flag
Example: `
$ meroxa resources create store --type postgres -u "$DATABASE_URL" --metadata '{"logical_replication":"true"}'
$ meroxa resources create datalake --type s3 -u "s3://$AWS_ACCESS_KEY_ID:$AWS_ACCESS_KEY_SECRET@us-east-1/meroxa-demos"
$ meroxa resources create warehouse --type redshift -u "$REDSHIFT_URL"
$ meroxa resources create slack --type url -u "$WEBHOOK_URL"
$ meroxa resource create mysqldb \
--type mysql \
--url "mysql://$MYSQL_USER:$MYSQL_PASS@$MYSQL_URL:$MYSQL_PORT/$MYSQL_DB"
$ meroxa resource create mybigquery \
--type bigquery \
-u "bigquery://$GCP_PROJECT_ID/$GCP_DATASET_NAME" \
--client-key "$(cat $GCP_SERVICE_ACCOUNT_JSON_FILE)"

$ meroxa resource create mongo \
--type mongodb \
-u "mongodb://$MONGO_USER:$MONGO_PASS@$MONGO_URL:$MONGO_PORT"
$ meroxa resource create sourcedb \
--type confluentcloud \
--url kafka+sasl+ssl://$API_KEY:$API_SECRET@<$BOOTSTRAP_SERVER>?sasl_mechanism=plain

$ meroxa resource create meteor \
--type cosmosdb \
--url cosmosdb://user:pass@org.documents.azure.com:443/pluto

$ meroxa resource create elasticsearch \
--type elasticsearch \
-u "https://$ES_USER:$ES_PASS@$ES_URL:$ES_PORT" \
--metadata '{"index.prefix": "$ES_INDEX","incrementing.field.name": "$ES_INCREMENTING_FIELD"}'

$ meroxa resource create mybigquery \
--type bigquery \
-u "bigquery://$GCP_PROJECT_ID/$GCP_DATASET_NAME" \
--client-key "$(cat $GCP_SERVICE_ACCOUNT_JSON_FILE)"
$ meroxa resource create sourcedb \
--type kafka \
--url kafka+sasl+ssl://$KAFKA_USER:$KAFKA_PASS@<$BOOTSTRAP_SERVER>?sasl_mechanism=plain

$ meroxa resource create mongo \
--type mongodb \
-u "mongodb://$MONGO_USER:$MONGO_PASS@$MONGO_URL:$MONGO_PORT"

$ meroxa resource create mysqldb \
--type mysql \
--url "mysql://$MYSQL_USER:$MYSQL_PASS@$MYSQL_URL:$MYSQL_PORT/$MYSQL_DB"

$ meroxa resource create workspace \
--type notion \
--token AbCdEfG123456

$ meroxa resource create workspace \
--type oracledb \
--url oracle://user:password@host.com:1521/database

$ meroxa resources create store \
--type postgres \
-u "$DATABASE_URL" \
--metadata '{"logical_replication":"true"}'

$ meroxa resources create warehouse \
--type redshift \
-u "$REDSHIFT_URL" \
--ssh-url ssh://user@password@example.elb.us-east-1.amazonaws.com:22 \
--private-key-file ~/.ssh/my-key

$ meroxa resources create datalake \
--type s3 \
-u "s3://$AWS_ACCESS_KEY_ID:$AWS_ACCESS_KEY_SECRET@us-east-1/meroxa-demos"

$ meroxa resource create snowflake \
--type snowflakedb \
-u "snowflake://$SNOWFLAKE_URL/meroxa_db/stream_data" \
--username meroxa_user \
--password $SNOWFLAKE_PRIVATE_KEY
--private-key-file /Users/me/.ssh/snowflake_ed25519

$ meroxa resource create sourcedb \
--type kafka \
--url kafka+sasl+ssl://$KAFKA_USER:$KAFKA_PASS@<$BOOTSTRAP_SERVER>?sasl_mechanism=plain
$ meroxa resource create hr \
--type sqlserver \
--url "sqlserver://$MSSQL_USER:$MSSQL_PASS@$MSSQL_URL:$MSSQL_PORT/$MSSQL_DB"

$ meroxa resource create sourcedb \
--type confluentcloud \
--url kafka+sasl+ssl://$API_KEY:$API_SECRET@<$BOOTSTRAP_SERVER>?sasl_mechanism=plain`,
$ meroxa resources create slack \
--type url \
-u "$WEBHOOK_URL"`,
}
}

Expand Down Expand Up @@ -177,6 +210,10 @@ func (c *Create) Execute(ctx context.Context) error {
env = string(meroxa.EnvironmentTypeCommon)
}

if err := c.handlePrivateKeyFlags(ctx); err != nil {
return err
}

if c.hasCredentials() {
input.Credentials = &meroxa.Credentials{
Username: c.flags.Username,
Expand Down Expand Up @@ -233,3 +270,25 @@ func (c *Create) hasCredentials() bool {
c.flags.Token != "" ||
c.flags.SSL
}

func (c *Create) handlePrivateKeyFlags(ctx context.Context) error {
path := c.flags.PrivateKeyFile
if path != "" && c.flags.SSHPrivateKey == "" {
bytes, err := os.ReadFile(path)
if err != nil {
return fmt.Errorf("could not find SSH private key at %q."+
" Try a different path", path)
}
key := string(bytes)
c.flags.SSHPrivateKey = key

if c.flags.Type == string(meroxa.ResourceTypeSnowflake) {
if c.flags.Password != "" {
c.logger.Warnf(ctx, "ignoring value of --ssh-private-key-file (%s) in favor of value of --password", c.flags.PrivateKeyFile)
} else {
c.flags.Password = key
}
}
}
return nil
}
108 changes: 108 additions & 0 deletions cmd/meroxa/root/resources/create_test.go
View file
Open in desktop
Expand Up @@ -4,10 +4,15 @@ import (
"context"
"encoding/json"
"fmt"
"os"
"path/filepath"
"reflect"
"testing"

"github.com/golang/mock/gomock"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

"github.com/meroxa/cli/cmd/meroxa/builder"
"github.com/meroxa/cli/cmd/meroxa/global"
Expand Down Expand Up @@ -59,6 +64,9 @@ func TestCreateResourceFlags(t *testing.T) {
{name: "metadata", required: false, shorthand: "m"},
{name: "env", required: false},
{name: "token", required: false},
{name: "ssh-url", required: false},
{name: "ssh-private-key", required: false},
{name: "private-key-file", required: false},
}

c := builder.BuildCobraCommand(&Create{})
Expand Down Expand Up @@ -358,3 +366,103 @@ Sign up for the Beta here: https://share.hsforms.com/1Uq6UYoL8Q6eV5QzSiyIQkAc2sm
t.Fatalf("expected output:\n%s\ngot:\n%s", wantError, gotError)
}
}

func TestCreateResourceExecutionPrivateKeyFlags(t *testing.T) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nice test coverage as always! ✨

ctx := context.Background()
logger := log.NewTestLogger()

keyVal := "super-secret"
keyFile := filepath.Join("/tmp", uuid.NewString())
err := os.WriteFile(keyFile, []byte(keyVal), 0600)
require.NoError(t, err)

tests := []struct {
name string
inputType string
inputSSHPrivateKeyFlag string
inputPasswordFlag string
inputPrivateKeyFileFlag string
expectedPassword string
expectedSSHPrivateKeyVal string
}{
{
name: "create postgres resource with SSH Tunnel --ssh-private-key",
inputType: string(meroxa.ResourceTypePostgres),
inputSSHPrivateKeyFlag: keyVal,
expectedPassword: "",
expectedSSHPrivateKeyVal: keyVal,
},
{
name: "create postgres resource with SSH Tunnel --private-key-file",
inputType: string(meroxa.ResourceTypePostgres),
inputPrivateKeyFileFlag: keyFile,
expectedPassword: "",
expectedSSHPrivateKeyVal: keyVal,
},
{
name: "create postgres resource with both SSH flags",
inputType: string(meroxa.ResourceTypePostgres),
inputPrivateKeyFileFlag: keyFile,
inputSSHPrivateKeyFlag: keyVal,
expectedPassword: "",
expectedSSHPrivateKeyVal: keyVal,
},
{
name: "create snowflake resource with --password",
inputType: string(meroxa.ResourceTypeSnowflake),
inputPasswordFlag: keyVal,
expectedPassword: keyVal,
expectedSSHPrivateKeyVal: "",
},
{
name: "create snowflake resource with --private-key-file",
inputPrivateKeyFileFlag: keyFile,
inputType: string(meroxa.ResourceTypeSnowflake),
expectedPassword: keyVal,
expectedSSHPrivateKeyVal: keyVal,
},
{
name: "create snowflake resource with both secret flags",
inputPasswordFlag: keyVal,
inputPrivateKeyFileFlag: keyFile,
inputType: string(meroxa.ResourceTypeSnowflake),
expectedPassword: keyVal,
expectedSSHPrivateKeyVal: keyVal,
},
}

for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
ctrl := gomock.NewController(t)
client := mock.NewMockClient(ctrl)

c := &Create{
client: client,
logger: logger,
}

client.
EXPECT().
CreateResource(
ctx,
gomock.Any(),
).
Return(&meroxa.Resource{}, nil)

c.args.Name = "my-resource"
c.flags.Type = tc.inputType
c.flags.URL = "anything"
c.flags.Password = tc.inputPasswordFlag
c.flags.SSHPrivateKey = tc.inputSSHPrivateKeyFlag
c.flags.PrivateKeyFile = tc.inputPrivateKeyFileFlag

err := c.Execute(ctx)
if err != nil {
t.Fatalf("not expected error, got %q", err.Error())
}

assert.Equalf(t, tc.expectedSSHPrivateKeyVal, c.flags.SSHPrivateKey, "mistach in private key flag value")
assert.Equalf(t, tc.expectedPassword, c.flags.Password, "mismatch in password flag value")
})
}
}