From c34c708265cb86f806dcc128395c6e73615fa208 Mon Sep 17 00:00:00 2001 From: Francisco Pombal Date: Fri, 9 Aug 2024 18:18:12 +0100 Subject: [PATCH 01/10] Add 2FA test and 2FA support for integration tests - Credentials for non 2FA/MFA-enabled accounts as well as 2FA/MFA-enabled accounts are supplied from the environment. - For tests that can use both types of credentials, credentials for non 2FA/MFA-enabled accounts are preferred, if available. - Tests that require one or the other type of credentials/accounts are skipped if the required type of credentials is not available. - A new test-only dependency is introduced for generating 2FA/MFA codes from their secrets: `github.com/pquerna/otp@v1.4.0`. - `golang.org/x/crypto` has been bumped to the latest version. --- go.mod | 6 ++- go.sum | 59 ++++++++++++++++++++++++--- mega_test.go | 111 +++++++++++++++++++++++++++++++++++++++++++++++---- 3 files changed, 160 insertions(+), 16 deletions(-) diff --git a/go.mod b/go.mod index d97bbf6..744bd14 100644 --- a/go.mod +++ b/go.mod @@ -1,5 +1,7 @@ module github.com/t3rm1n4l/go-mega -require golang.org/x/crypto v0.1.0 - go 1.13 + +require golang.org/x/crypto v0.26.0 + +require github.com/pquerna/otp v1.4.0 diff --git a/go.sum b/go.sum index ae56425..fe04afc 100644 --- a/go.sum +++ b/go.sum @@ -1,29 +1,76 @@ +github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc h1:biVzkmvwrH8WK8raXaxBx6fRVTlJILwEwQGL1I/ByEI= +github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pquerna/otp v1.4.0 h1:wZvl1TIVxKRThZIBiwOOHOGP/1+nZyWBil9Y2XNEDzg= +github.com/pquerna/otp v1.4.0/go.mod h1:dkJfzwRKNiegxyNb54X/3fLwhCynbMspSyWKnvi1AEg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= -golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= +golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/mega_test.go b/mega_test.go index cea481e..45e32aa 100644 --- a/mega_test.go +++ b/mega_test.go @@ -10,10 +10,17 @@ import ( "sync" "testing" "time" + + "github.com/pquerna/otp/totp" ) +// Credentials for non MFA-enabled accounts var USER string = os.Getenv("MEGA_USER") var PASSWORD string = os.Getenv("MEGA_PASSWD") +// Credentials for MFA-enabled accounts +var USER_MFA = os.Getenv("MEGA_USER_MFA") +var PASSWORD_MFA string = os.Getenv("MEGA_PASSWD_MFA") +var SECRET_MFA string = os.Getenv("MEGA_SECRET_MFA") // retry runs fn until it succeeds, using what to log and retrying on // EAGAIN. It uses exponential backoff @@ -36,18 +43,91 @@ func retry(t *testing.T, what string, fn func() error) { t.Fatalf("%s failed: %v", what, err) } -func skipIfNoCredentials(t *testing.T) { +type CredentialType int + +const ( + Credentials CredentialType = iota + MfaCredentials + AnyCredentials +) + +// getMfaCode generates an MFA code using the provided secret and the current time. +// If the code cannot be generated, it returns an error. +func getMfaCode(secret string) (string, error) { + return totp.GenerateCode(secret, time.Now()) +} + +// getCredentials retrieves credentials for an MFA-enabled account from predefined variables set from the environment. +// If either the user, password or MFA secret are not set, or if an MFA code cannot be successfully generated from the given secret, it returns an error indicating that the credentials are missing or invalid. +func getMfaCredentials() (string, string, string, error) { + if USER_MFA == "" || PASSWORD_MFA == "" || SECRET_MFA == "" { + return "", "", "", fmt.Errorf("MEGA_USER_MFA, MEGA_PASSWD_MFA or MEGA_SECRET_MFA not set.") + } + + mfa_code, mfa_code_err := getMfaCode(SECRET_MFA) + if mfa_code_err != nil { + return "", "", "", fmt.Errorf("Generating MFA code failed: %w", mfa_code_err) + } + + return USER_MFA, PASSWORD_MFA, mfa_code, nil +} + +// getCredentials retrieves credentials for a non MFA-enabled account from predefined variables set from the environment. +// If either the user or password are not set, it returns an error indicating that the credentials are missing. +func getCredentials() (string, string, error) { if USER == "" || PASSWORD == "" { - t.Skip("MEGA_USER and MEGA_PASSWD not set - skipping integration tests") + return "", "", fmt.Errorf("MEGA_USER or MEGA_PASSWD not set.") } + return USER, PASSWORD, nil +} + +// getCredentialsOrSkip retrieves user credentials based on the specified CredentialType. +// It supports both standard and MFA-enabled credentials. +// If the requested credentials are missing or invalid, the function skips the test with an appropriate message. +func getCredentialsOrSkip(t *testing.T, credentialsType CredentialType) (string, string, string) { + + switch credentialsType { + case Credentials: + user, password, err := getCredentials() + if err != nil { + t.Skipf("Skipping test due to credentials error: %v", err) + } + return user, password, "" + case MfaCredentials: + user, password, mfa_code, err := getMfaCredentials() + if err != nil { + t.Skipf("Skipping test due to credentials error: %v", err) + } + return user, password, mfa_code + case AnyCredentials: + user, password, err := getCredentials() + if err != nil { + t.Logf("Trying with MFA credentials instead, getting other credentials failed with error: %v", err) + } else { + return user, password, "" + } + user, password, mfa_code, err := getMfaCredentials() + if err != nil { + t.Skipf("Skipping test due to credentials error: %v", err) + } + return user, password, mfa_code + default: + t.Fatal("Invalid credentials type") + } + + t.Fatal("Unreachable!") + return "", "", "" } func initSession(t *testing.T) *Mega { - skipIfNoCredentials(t) + user, password, mfa_code := getCredentialsOrSkip(t, AnyCredentials) m := New() // m.SetDebugger(log.Printf) retry(t, "Login", func() error { - return m.Login(USER, PASSWORD) + if mfa_code != "" { + return m.MultiFactorLogin(user, password, mfa_code) + } + return m.Login(user, password) }) return m } @@ -121,11 +201,20 @@ func fileMD5(t *testing.T, name string) string { } func TestLogin(t *testing.T) { - skipIfNoCredentials(t) + user, password, _ := getCredentialsOrSkip(t, Credentials) m := New() retry(t, "Login", func() error { - return m.Login(USER, PASSWORD) + return m.Login(user, password) + }) +} + +func TestMfaLogin(t *testing.T) { + user, password, mfa_code := getCredentialsOrSkip(t, MfaCredentials) + + m := New() + retry(t, "MfaLogin", func() error { + return m.MultiFactorLogin(user, password, mfa_code) }) } @@ -254,11 +343,17 @@ func TestCreateDir(t *testing.T) { } func TestConfig(t *testing.T) { - skipIfNoCredentials(t) + user, password, mfa_code := getCredentialsOrSkip(t, AnyCredentials) m := New() m.SetAPIUrl("http://invalid.domain") - err := m.Login(USER, PASSWORD) + err := func() error { + if mfa_code != "" { + return m.MultiFactorLogin(user, password, mfa_code) + } + return m.Login(user, password) + }() + if err == nil { t.Error("API Url: Expected failure") } From aa4fcea6b9ea251adfec7566d333b0f36059fed9 Mon Sep 17 00:00:00 2001 From: Francisco Pombal Date: Mon, 16 Sep 2024 23:08:55 +0100 Subject: [PATCH 02/10] Implement logout API command/request - Move command definitions to a separate file. - Clean up sessions in most cases in tests with the new logout command. Sessions are only properly terminated in successful runs of tests. --- commands.go | 19 +++++++++++++++++++ mega.go | 43 ++++++++++++++++++++++++++++++------------- mega_test.go | 26 ++++++++++++++++++++++++++ messages.go | 31 ++++++++++++++++++------------- 4 files changed, 93 insertions(+), 26 deletions(-) create mode 100644 commands.go diff --git a/commands.go b/commands.go new file mode 100644 index 0000000..3c89112 --- /dev/null +++ b/commands.go @@ -0,0 +1,19 @@ +package mega + +type APICommand string + +const ( + COMMAND_PRELOGIN APICommand = "us0" + COMMAND_LOGIN APICommand = "us" + COMMAND_LOGOUT APICommand = "sml" + COMMAND_GET_USER APICommand = "ug" + COMMAND_GET_USER_QUOTA APICommand = "uq" + COMMAND_FILES APICommand = "f" + COMMAND_DOWNLOAD APICommand = "g" + COMMAND_UPLOAD APICommand = "u" + COMMAND_UPLOAD_COMPLETE APICommand = "p" + COMMAND_MOVE APICommand = "m" + COMMAND_RENAME APICommand = "a" + COMMAND_DELETE APICommand = "d" + COMMAND_GET_LINK APICommand = "l" +) diff --git a/mega.go b/mega.go index 1bd5f1c..bf540f3 100644 --- a/mega.go +++ b/mega.go @@ -482,7 +482,7 @@ func (m *Mega) prelogin(email string) error { email = strings.ToLower(email) // mega uses lowercased emails for login purposes - FIXME is this true for prelogin? - msg[0].Cmd = "us0" + msg[0].Cmd = COMMAND_PRELOGIN msg[0].User = email req, err := json.Marshal(msg) @@ -537,7 +537,7 @@ func (m *Mega) login(email string, passwd string, multiFactor string) error { m.uh = make([]byte, len(uhandle)) copy(m.uh, uhandle) - msg[0].Cmd = "us" + msg[0].Cmd = COMMAND_LOGIN msg[0].User = email msg[0].Mfa = multiFactor @@ -618,6 +618,23 @@ func (m *Mega) MultiFactorLogin(email, passwd, multiFactor string) error { return nil } +func (m *Mega) Logout() error { + var msg [1]LogoutMsg + + msg[0].Cmd = COMMAND_LOGOUT + + req, err := json.Marshal(msg) + if err != nil { + return err + } + _, err = m.api_request(req) + if err != nil { + return err + } + + return nil +} + // WaitEventsStart - call this before you do the action which might // generate events then use the returned channel as a parameter to // WaitEvents to wait for the event(s) to be received. @@ -667,7 +684,7 @@ func (m *Mega) GetUser() (UserResp, error) { var msg [1]UserMsg var res [1]UserResp - msg[0].Cmd = "ug" + msg[0].Cmd = COMMAND_GET_USER req, err := json.Marshal(msg) if err != nil { @@ -687,7 +704,7 @@ func (m *Mega) GetQuota() (QuotaResp, error) { var msg [1]QuotaMsg var res [1]QuotaResp - msg[0].Cmd = "uq" + msg[0].Cmd = COMMAND_GET_USER_QUOTA msg[0].Xfer = 1 msg[0].Strg = 1 @@ -924,7 +941,7 @@ func (m *Mega) getFileSystem() error { var msg [1]FilesMsg var res [1]FilesResp - msg[0].Cmd = "f" + msg[0].Cmd = COMMAND_FILES msg[0].C = 1 req, err := json.Marshal(msg) @@ -990,7 +1007,7 @@ func (m *Mega) NewDownload(src *Node) (*Download, error) { var res [1]DownloadResp m.FS.mutex.Lock() - msg[0].Cmd = "g" + msg[0].Cmd = COMMAND_DOWNLOAD msg[0].G = 1 msg[0].N = src.hash if m.config.https { @@ -1313,7 +1330,7 @@ func (m *Mega) NewUpload(parent *Node, name string, fileSize int64) (*Upload, er var res [1]UploadResp parenthash := parent.GetHash() - msg[0].Cmd = "u" + msg[0].Cmd = COMMAND_UPLOAD msg[0].S = fileSize if m.config.https { msg[0].SSL = 2 @@ -1532,7 +1549,7 @@ func (u *Upload) Finish() (node *Node, err error) { var cmsg [1]UploadCompleteMsg var cres [1]UploadCompleteResp - cmsg[0].Cmd = "p" + cmsg[0].Cmd = COMMAND_UPLOAD_COMPLETE cmsg[0].T = u.parenthash cmsg[0].N[0].H = string(u.completion_handle) cmsg[0].N[0].T = FILE @@ -1667,7 +1684,7 @@ func (m *Mega) Move(src *Node, parent *Node) error { var msg [1]MoveFileMsg var err error - msg[0].Cmd = "m" + msg[0].Cmd = COMMAND_MOVE msg[0].N = src.hash msg[0].T = parent.hash msg[0].I, err = randString(10) @@ -1719,7 +1736,7 @@ func (m *Mega) Rename(src *Node, name string) error { return err } - msg[0].Cmd = "a" + msg[0].Cmd = COMMAND_RENAME msg[0].Attr = attr_data msg[0].Key = base64urlencode(key) msg[0].N = src.hash @@ -1777,7 +1794,7 @@ func (m *Mega) CreateDir(name string, parent *Node) (*Node, error) { return nil, err } - msg[0].Cmd = "p" + msg[0].Cmd = COMMAND_UPLOAD_COMPLETE msg[0].T = parent.hash msg[0].N[0].H = "xxxxxxxx" msg[0].N[0].T = FOLDER @@ -1820,7 +1837,7 @@ func (m *Mega) Delete(node *Node, destroy bool) error { var msg [1]FileDeleteMsg var err error - msg[0].Cmd = "d" + msg[0].Cmd = COMMAND_DELETE msg[0].N = node.hash msg[0].I, err = randString(10) if err != nil { @@ -2049,7 +2066,7 @@ func (m *Mega) getLink(n *Node) (string, error) { var msg [1]GetLinkMsg var res [1]string - msg[0].Cmd = "l" + msg[0].Cmd = COMMAND_GET_LINK msg[0].N = n.GetHash() req, err := json.Marshal(msg) diff --git a/mega_test.go b/mega_test.go index 45e32aa..b520290 100644 --- a/mega_test.go +++ b/mega_test.go @@ -132,6 +132,12 @@ func initSession(t *testing.T) *Mega { return m } +func endSession(t *testing.T, session *Mega) { + retry(t, "Logout", func() error { + return session.Logout() + }) +} + // createFile creates a temporary file of a given size along with its MD5SUM func createFile(t *testing.T, size int64) (string, string) { b := make([]byte, size) @@ -207,6 +213,8 @@ func TestLogin(t *testing.T) { retry(t, "Login", func() error { return m.Login(user, password) }) + + endSession(t, m) } func TestMfaLogin(t *testing.T) { @@ -216,6 +224,13 @@ func TestMfaLogin(t *testing.T) { retry(t, "MfaLogin", func() error { return m.MultiFactorLogin(user, password, mfa_code) }) + + endSession(t, m) +} + +func TestLogout(t *testing.T) { + session := initSession(t) + endSession(t, session) } func TestGetUser(t *testing.T) { @@ -224,6 +239,7 @@ func TestGetUser(t *testing.T) { if err != nil { t.Fatal("GetUser failed", err) } + endSession(t, session) } func TestUploadDownload(t *testing.T) { @@ -263,6 +279,7 @@ func TestUploadDownload(t *testing.T) { } } session.SetHTTPS(false) + endSession(t, session) } func TestMove(t *testing.T) { @@ -282,6 +299,8 @@ func TestMove(t *testing.T) { t.Error("Move happened to wrong parent", phash, n.parent.hash) } session.FS.mutex.Unlock() + + endSession(t, session) } func TestRename(t *testing.T) { @@ -299,6 +318,7 @@ func TestRename(t *testing.T) { t.Error("Renamed to wrong name", newname) } session.FS.mutex.Unlock() + endSession(t, session) } func TestDelete(t *testing.T) { @@ -327,6 +347,7 @@ func TestDelete(t *testing.T) { t.Error("Expects file to be dissapeared") } session.FS.mutex.Unlock() + endSession(t, session) } func TestCreateDir(t *testing.T) { @@ -340,6 +361,7 @@ func TestCreateDir(t *testing.T) { t.Error("Wrong directory parent") } session.FS.mutex.Unlock() + endSession(t, session) } func TestConfig(t *testing.T) { @@ -420,6 +442,7 @@ func TestPathLookup(t *testing.T) { } } } + endSession(t, session) } func TestEventNotify(t *testing.T) { @@ -449,6 +472,8 @@ func TestEventNotify(t *testing.T) { if node != nil { t.Fatal("Expects file to not-found in first client's FS") } + endSession(t, session1) + endSession(t, session2) } func TestExportLink(t *testing.T) { @@ -466,6 +491,7 @@ func TestExportLink(t *testing.T) { _, err := session.Link(node, true) return err }) + endSession(t, session) } func TestWaitEvents(t *testing.T) { diff --git a/messages.go b/messages.go index 9070a14..dbd101b 100644 --- a/messages.go +++ b/messages.go @@ -3,7 +3,7 @@ package mega import "encoding/json" type PreloginMsg struct { - Cmd string `json:"a"` + Cmd APICommand `json:"a"` User string `json:"user"` } @@ -13,7 +13,7 @@ type PreloginResp struct { } type LoginMsg struct { - Cmd string `json:"a"` + Cmd APICommand `json:"a"` User string `json:"user"` Handle string `json:"uh"` SessionKey string `json:"sek,omitempty"` @@ -30,8 +30,13 @@ type LoginResp struct { U string `json:"u"` } +type LogoutMsg struct { + // "a" should be "sml" for logout + Cmd APICommand `json:"a"` +} + type UserMsg struct { - Cmd string `json:"a"` + Cmd APICommand `json:"a"` } type UserResp struct { @@ -49,7 +54,7 @@ type UserResp struct { type QuotaMsg struct { // Action, should be "uq" for quota request - Cmd string `json:"a"` + Cmd APICommand `json:"a"` // xfer should be 1 Xfer int `json:"xfer"` // Without strg=1 only reports total capacity for account @@ -66,7 +71,7 @@ type QuotaResp struct { } type FilesMsg struct { - Cmd string `json:"a"` + Cmd APICommand `json:"a"` C int `json:"c"` } @@ -108,12 +113,12 @@ type FileAttr struct { } type GetLinkMsg struct { - Cmd string `json:"a"` + Cmd APICommand `json:"a"` N string `json:"n"` } type DownloadMsg struct { - Cmd string `json:"a"` + Cmd APICommand `json:"a"` G int `json:"g"` P string `json:"p,omitempty"` N string `json:"n,omitempty"` @@ -128,7 +133,7 @@ type DownloadResp struct { } type UploadMsg struct { - Cmd string `json:"a"` + Cmd APICommand `json:"a"` S int64 `json:"s"` SSL int `json:"ssl,omitempty"` } @@ -138,7 +143,7 @@ type UploadResp struct { } type UploadCompleteMsg struct { - Cmd string `json:"a"` + Cmd APICommand `json:"a"` T string `json:"t"` N [1]struct { H string `json:"h"` @@ -154,20 +159,20 @@ type UploadCompleteResp struct { } type FileInfoMsg struct { - Cmd string `json:"a"` + Cmd APICommand `json:"a"` F int `json:"f"` P string `json:"p"` } type MoveFileMsg struct { - Cmd string `json:"a"` + Cmd APICommand `json:"a"` N string `json:"n"` T string `json:"t"` I string `json:"i"` } type FileAttrMsg struct { - Cmd string `json:"a"` + Cmd APICommand `json:"a"` Attr string `json:"attr"` Key string `json:"key"` N string `json:"n"` @@ -175,7 +180,7 @@ type FileAttrMsg struct { } type FileDeleteMsg struct { - Cmd string `json:"a"` + Cmd APICommand `json:"a"` N string `json:"n"` I string `json:"i"` } From ae67e446644311905da3f545bdc1dd52e8152f15 Mon Sep 17 00:00:00 2001 From: Francisco Pombal Date: Mon, 16 Sep 2024 23:09:25 +0100 Subject: [PATCH 03/10] Better handling of unimplemented server events --- errors.go | 3 +++ mega.go | 27 +++++++++++++++++++++++---- messages.go | 4 ++-- 3 files changed, 28 insertions(+), 6 deletions(-) diff --git a/errors.go b/errors.go index 2c02fb5..af1fd18 100644 --- a/errors.go +++ b/errors.go @@ -42,6 +42,9 @@ var ( // Config errors EWORKER_LIMIT_EXCEEDED = errors.New("Maximum worker limit exceeded") + + // Unimplemented + ENOTIMPLEMENTED = errors.New("Not implemented") ) type ErrorMsg int diff --git a/mega.go b/mega.go index bf540f3..d365404 100644 --- a/mega.go +++ b/mega.go @@ -1925,6 +1925,10 @@ func (m *Mega) processDeleteNode(evRaw []byte) error { return nil } +func (m *Mega) processEventStub(evRaw []byte) error { + return ENOTIMPLEMENTED +} + // Listen for server event notifications and play actions func (m *Mega) pollEvents() { var err error @@ -2021,11 +2025,11 @@ func (m *Mega) pollEvents() { m.logf("pollEvents: Couldn't parse event from server: %v: %s", err, evRaw) continue } - m.debugf("pollEvents: Parsing event %q: %s", gev.Cmd, evRaw) + m.debugf("pollEvents: Parsing event %q: %s", gev.GEventType, evRaw) // Work out what to do with the event var process func([]byte) error - switch gev.Cmd { + switch gev.GEventType { case "t": // node addition process = m.processAddNode case "u": // node update @@ -2033,29 +2037,44 @@ func (m *Mega) pollEvents() { case "d": // node deletion process = m.processDeleteNode case "s", "s2": // share addition/update/revocation + process = m.processEventStub case "c": // contact addition/update + process = m.processEventStub case "k": // crypto key request + process = m.processEventStub case "fa": // file attribute update + process = m.processEventStub case "ua": // user attribute update + process = m.processEventStub case "psts": // account updated + process = m.processEventStub case "ipc": // incoming pending contact request (to us) + process = m.processEventStub case "opc": // outgoing pending contact request (from us) + process = m.processEventStub case "upci": // incoming pending contact request update (accept/deny/ignore) + process = m.processEventStub case "upco": // outgoing pending contact request update (from them, accept/deny/ignore) + process = m.processEventStub case "ph": // public links handles + process = m.processEventStub case "se": // set email + process = m.processEventStub case "mcc": // chat creation / peer's invitation / peer's removal + process = m.processEventStub case "mcna": // granted / revoked access to a node + process = m.processEventStub case "uac": // user access control + process = m.processEventStub default: - m.debugf("pollEvents: Unknown message %q received: %s", gev.Cmd, evRaw) + m.debugf("pollEvents: Unknown message %q received: %s", gev.GEventType, evRaw) } // process the event if we can if process != nil { err := process(evRaw) if err != nil { - m.logf("pollEvents: Error processing event %q '%s': %v", gev.Cmd, evRaw, err) + m.logf("pollEvents: Error processing event %q '%s': %v", gev.GEventType, evRaw, err) } } } diff --git a/messages.go b/messages.go index dbd101b..c866c42 100644 --- a/messages.go +++ b/messages.go @@ -188,7 +188,7 @@ type FileDeleteMsg struct { // GenericEvent is a generic event for parsing the Cmd type before // decoding more specifically type GenericEvent struct { - Cmd string `json:"a"` + GEventType string `json:"a"` } // FSEvent - event for various file system events @@ -197,7 +197,7 @@ type GenericEvent struct { // Update attr (a=u) // New nodes (a=t) type FSEvent struct { - Cmd string `json:"a"` + FSEventType string `json:"a"` T struct { Files []FSNode `json:"f"` From ef8ec550d5d4411e9147b16f5a3446ea354e5bb5 Mon Sep 17 00:00:00 2001 From: Francisco Pombal Date: Tue, 17 Sep 2024 12:33:43 +0100 Subject: [PATCH 04/10] Fix HTTP client parameters - Fix timeout override. - Override User-Agent header. --- mega.go | 5 ++++- utils.go | 39 +++++++++++++++++++++++---------------- 2 files changed, 27 insertions(+), 17 deletions(-) diff --git a/mega.go b/mega.go index d365404..1eedca4 100644 --- a/mega.go +++ b/mega.go @@ -28,6 +28,7 @@ import ( const ( API_URL = "https://g.api.mega.co.nz" BASE_DOWNLOAD_URL = "https://mega.co.nz" + USER_AGENT = "GoMega/1.0" RETRIES = 10 DOWNLOAD_WORKERS = 3 MAX_DOWNLOAD_WORKERS = 30 @@ -41,6 +42,7 @@ const ( type config struct { baseurl string + useragent string retries int dl_workers int ul_workers int @@ -51,6 +53,7 @@ type config struct { func newConfig() config { return config{ baseurl: API_URL, + useragent: USER_AGENT, retries: RETRIES, dl_workers: DOWNLOAD_WORKERS, ul_workers: UPLOAD_WORKERS, @@ -354,7 +357,7 @@ func New() *Mega { config: cfg, sn: bigx.Int64(), FS: mgfs, - client: newHttpClient(cfg.timeout), + client: newHttpClient(cfg.useragent, cfg.timeout), } m.SetLogger(log.Printf) m.SetDebugger(nil) diff --git a/utils.go b/utils.go index 0660c36..abf299e 100644 --- a/utils.go +++ b/utils.go @@ -10,29 +10,36 @@ import ( "encoding/json" "errors" "math/big" - "net" "net/http" + "net/url" "regexp" "strings" "time" ) -func newHttpClient(timeout time.Duration) *http.Client { - // TODO: Need to test this out - // Doesn't seem to work as expected - c := &http.Client{ - Transport: &http.Transport{ - Dial: func(netw, addr string) (net.Conn, error) { - c, err := net.DialTimeout(netw, addr, timeout) - if err != nil { - return nil, err - } - return c, nil - }, +type CustomTransport struct { + Transport http.RoundTripper + Proxy func(*http.Request) (*url.URL, error) + UserAgent string +} + +func (t *CustomTransport) RoundTrip(req *http.Request) (*http.Response, error) { + req.Header.Set("User-Agent", t.UserAgent) + return t.Transport.RoundTrip(req) +} + +func newHttpClient(userAgent string, timeout time.Duration) *http.Client { + + client := &http.Client{ + Transport: &CustomTransport{ + Transport: http.DefaultTransport, Proxy: http.ProxyFromEnvironment, - }, - } - return c + UserAgent: userAgent, + }, + Timeout: timeout, + } + + return client } // bytes_to_a32 converts the byte slice b to uint32 slice considering From 464a967b7c745855a4a27ae7aa0221e1c48a2309 Mon Sep 17 00:00:00 2001 From: Francisco Pombal Date: Tue, 17 Sep 2024 14:02:20 +0100 Subject: [PATCH 05/10] Implement get user sessions API command/request --- commands.go | 1 + mega.go | 24 ++++++++++++++++++++++++ mega_test.go | 9 +++++++++ messages.go | 38 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 72 insertions(+) diff --git a/commands.go b/commands.go index 3c89112..4b1d527 100644 --- a/commands.go +++ b/commands.go @@ -16,4 +16,5 @@ const ( COMMAND_RENAME APICommand = "a" COMMAND_DELETE APICommand = "d" COMMAND_GET_LINK APICommand = "l" + COMMAND_GET_USER_SESSIONS APICommand = "usl" ) diff --git a/mega.go b/mega.go index 1eedca4..940a36b 100644 --- a/mega.go +++ b/mega.go @@ -1863,6 +1863,30 @@ func (m *Mega) Delete(node *Node, destroy bool) error { return nil } +func (m *Mega) GetUserSessions() ([]GetUserSessionsResp, error) { + var msg [1]GetUserSessionsMsg + + msg[0].Cmd = COMMAND_GET_USER_SESSIONS + msg[0].IdAndAliveInfo = 1 + msg[0].DeviceIDInfo = 1 + + request, err := json.Marshal(msg) + if err != nil { + return nil, err + } + result, err := m.api_request(request) + if err != nil { + return nil, err + } + + var res [1][]GetUserSessionsResp + if err := json.Unmarshal(result, &res); err != nil { + return nil, err + } + + return res[0], nil +} + // process an add node event func (m *Mega) processAddNode(evRaw []byte) error { m.FS.mutex.Lock() diff --git a/mega_test.go b/mega_test.go index b520290..550d22f 100644 --- a/mega_test.go +++ b/mega_test.go @@ -350,6 +350,15 @@ func TestDelete(t *testing.T) { endSession(t, session) } +func TestGetUserSessions(t *testing.T) { + session := initSession(t) + _, err := session.GetUserSessions() + if err != nil { + t.Fatal("GetUserSessions failed", err) + } + endSession(t, session) +} + func TestCreateDir(t *testing.T) { session := initSession(t) node := createDir(t, session, "testdir1", session.FS.root) diff --git a/messages.go b/messages.go index c866c42..e852c9d 100644 --- a/messages.go +++ b/messages.go @@ -185,6 +185,44 @@ type FileDeleteMsg struct { I string `json:"i"` } +type GetUserSessionsMsg struct { + Cmd APICommand `json:"a"` + IdAndAliveInfo int `json:"x"` + DeviceIDInfo int `json:"d"` +} + +type GetUserSessionsResp struct { + DateTime int + Unused int + UserAgent string + IPAddress string + Country string + IsCurrent int + SessionID string + IsActive int + DeviceID int +} + +type getUserSessionsError struct { + message string +} + +func (e *getUserSessionsError) Error() string { + return e.message +} + +func (n *GetUserSessionsResp) UnmarshalJSON(buf []byte) error { + tmp := []interface{}{&n.DateTime, &n.Unused, &n.UserAgent, &n.IPAddress, &n.Country, &n.IsCurrent, &n.SessionID, &n.IsActive, &n.DeviceID} + wantLen := len(tmp) + if err := json.Unmarshal(buf, &tmp); err != nil { + return err + } + if g, e := len(tmp), wantLen; g != e { + return &getUserSessionsError{message: "wrong number of fields in GetUserSessionsResp"} + } + return nil +} + // GenericEvent is a generic event for parsing the Cmd type before // decoding more specifically type GenericEvent struct { From 9575f6351c01dd125206e7ea8f0d65c81f019c18 Mon Sep 17 00:00:00 2001 From: Francisco Pombal Date: Fri, 20 Sep 2024 10:41:14 +0100 Subject: [PATCH 06/10] Move away from the deprecated `io/ioutil` package --- mega.go | 9 ++++----- mega_test.go | 7 ++++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/mega.go b/mega.go index 940a36b..7ea1cb7 100644 --- a/mega.go +++ b/mega.go @@ -10,7 +10,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "log" "math/big" mrand "math/rand" @@ -438,7 +437,7 @@ func (m *Mega) api_request(r []byte) (buf []byte, err error) { _ = resp.Body.Close() continue } - buf, err = ioutil.ReadAll(resp.Body) + buf, err = io.ReadAll(resp.Body) if err != nil { _ = resp.Body.Close() continue @@ -1129,7 +1128,7 @@ func (d *Download) DownloadChunk(id int) (chunk []byte, err error) { return nil, errors.New("retries exceeded") } - chunk, err = ioutil.ReadAll(resp.Body) + chunk, err = io.ReadAll(resp.Body) if err != nil { _ = resp.Body.Close() return nil, err @@ -1484,7 +1483,7 @@ func (u *Upload) UploadChunk(id int, chunk []byte) (err error) { return errors.New("retries exceeded") } - chunk_resp, err = ioutil.ReadAll(rsp.Body) + chunk_resp, err = io.ReadAll(rsp.Body) if err != nil { _ = rsp.Body.Close() return err @@ -1983,7 +1982,7 @@ func (m *Mega) pollEvents() { continue } - buf, err := ioutil.ReadAll(resp.Body) + buf, err := io.ReadAll(resp.Body) if err != nil { m.logf("pollEvents: Error reading body: %v", err) _ = resp.Body.Close() diff --git a/mega_test.go b/mega_test.go index 550d22f..1c3ca04 100644 --- a/mega_test.go +++ b/mega_test.go @@ -4,7 +4,7 @@ import ( "crypto/md5" "crypto/rand" "fmt" - "io/ioutil" + "io" "os" "path" "sync" @@ -17,6 +17,7 @@ import ( // Credentials for non MFA-enabled accounts var USER string = os.Getenv("MEGA_USER") var PASSWORD string = os.Getenv("MEGA_PASSWD") + // Credentials for MFA-enabled accounts var USER_MFA = os.Getenv("MEGA_USER_MFA") var PASSWORD_MFA string = os.Getenv("MEGA_PASSWD_MFA") @@ -145,7 +146,7 @@ func createFile(t *testing.T, size int64) (string, string) { if err != nil { t.Fatalf("Error reading rand: %v", err) } - file, err := ioutil.TempFile("/tmp/", "gomega-") + file, err:= os.CreateTemp("/tmp/", "gomega-") if err != nil { t.Fatalf("Error creating temp file: %v", err) } @@ -194,7 +195,7 @@ func fileMD5(t *testing.T, name string) string { if err != nil { t.Fatalf("Failed to open %q: %v", name, err) } - b, err := ioutil.ReadAll(file) + b, err := io.ReadAll(file) if err != nil { t.Fatalf("Failed to read all %q: %v", name, err) } From f1cd3d4d5e5d6fd65c1a7e4661a40a10d759273a Mon Sep 17 00:00:00 2001 From: Francisco Pombal Date: Fri, 20 Sep 2024 10:44:37 +0100 Subject: [PATCH 07/10] Save the session key on logins It's the `sek` field in login responses. This is necessary for the upcoming "dump session" implementation. --- mega.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mega.go b/mega.go index 7ea1cb7..730bafe 100644 --- a/mega.go +++ b/mega.go @@ -116,6 +116,8 @@ type Mega struct { ssn string // Session ID sid string + // Session key + sek []byte // Master key k []byte // User handle @@ -574,6 +576,11 @@ func (m *Mega) login(email string, passwd string, multiFactor string) error { return err } + m.sek, err = base64urldecode(res[0].SessionKey) + if err != nil { + return err + } + m.k, err = base64urldecode(res[0].Key) if err != nil { return err From f5ecdfd00939a06664708d25fafcc370e89dabcc Mon Sep 17 00:00:00 2001 From: Francisco Pombal Date: Fri, 20 Sep 2024 10:48:55 +0100 Subject: [PATCH 08/10] Implement session logins and ability to dump session The session dump string is the same as one obtained from running the `session` command of `mega-cmd`. Currently, only session version 1 (normal login session) is supported. --- mega.go | 110 ++++++++++++++++++++++++++++++++++++++++++++++++++++ messages.go | 12 ++++++ 2 files changed, 122 insertions(+) diff --git a/mega.go b/mega.go index 730bafe..e2de7f3 100644 --- a/mega.go +++ b/mega.go @@ -627,6 +627,116 @@ func (m *Mega) MultiFactorLogin(email, passwd, multiFactor string) error { return nil } +func getRandomBytes() ([]byte, error) { + b := make([]byte, 16) + _, err := rand.Read(b) + if err != nil { + return nil, err + } + return b, nil +} + +func (m *Mega) SessionLogin(sessionString string) error { + + // decode the session + session, err := base64urldecode(sessionString) + if err != nil { + return err + } + // session is a byte array with the following format: [version, encryptedMasterKey, sid] + // version is 1 byte + // encryptedMasterKey is 16 bytes + // sid is the remaining bytes + sessionVersion := session[0] + // TODO: right now, only session version 1 (normal login session) is supported + if sessionVersion != 1 { + return errors.New("unsupported session version") + } + encryptedMasterKey := session[1:17] + sid := session[17:] + + // set the sid string + m.sid = base64urlencode(sid) + + // build the session login message with a generated session key + var msg [1]SessionLoginMsg + msg[0].Cmd = COMMAND_LOGIN + rb, err := getRandomBytes() + if err != nil { + return err + } + generatedSek := base64urlencode(rb) + msg[0].Sek = generatedSek + + // encode the message and send the request + req, err := json.Marshal(msg) + if err != nil { + return err + } + var result []byte + result, err = m.api_request(req) + if err != nil { + return err + } + + // decode the response + var res [1]SessionLoginResp + err = json.Unmarshal(result, &res) + if err != nil { + return err + } + + // set the user handle + m.uh = make([]byte, len(res[0].U)) + copy(m.uh, res[0].U) + + // set the session key + receivedSek, err := base64urldecode(res[0].SessionKey) + if err != nil { + return err + } + m.sek = make([]byte, len(receivedSek)) + copy(m.sek, receivedSek) + + // decrypt encrypted master key with session key + cipher, err := aes.NewCipher(m.sek) + if err != nil { + return err + } + m.k = make([]byte, 16) + cipher.Decrypt(m.k, encryptedMasterKey) + + // misc + waitEvent := m.WaitEventsStart() + err = m.getFileSystem() + if err != nil { + return err + } + // Wait until the all the pending events have been received + m.WaitEvents(waitEvent, 5*time.Second) + + return nil +} + +func (m *Mega) DumpSession() (string, error) { + sid, err := base64urldecode(m.sid) + if err != nil { + return "", err + } + + cipher, err := aes.NewCipher(m.sek) + if err != nil { + return "", err + } + encryptedMasterKey := make([]byte, 16) + cipher.Encrypt(encryptedMasterKey, m.k) + + // TODO: right now, only session version 1 (normal login session) is supported + session := append([]byte{1}, append(encryptedMasterKey, sid...)...) + + return base64urlencode(session), err +} + func (m *Mega) Logout() error { var msg [1]LogoutMsg diff --git a/messages.go b/messages.go index e852c9d..36ab059 100644 --- a/messages.go +++ b/messages.go @@ -30,6 +30,18 @@ type LoginResp struct { U string `json:"u"` } +type SessionLoginMsg struct { + Cmd APICommand `json:"a"` + Sek string `json:"sek"` +} + +type SessionLoginResp struct { + Privk string `json:"privk"` + Ach int `json:"ach"` + SessionKey string `json:"sek"` + U string `json:"u"` +} + type LogoutMsg struct { // "a" should be "sml" for logout Cmd APICommand `json:"a"` From daec23b85a7b0aa436db1c7e1c9f6389774eb4ac Mon Sep 17 00:00:00 2001 From: Francisco Pombal Date: Fri, 20 Sep 2024 10:57:08 +0100 Subject: [PATCH 09/10] Refactor tests to use session logins - This enables tests to run (and pass) without having to worry about expired or invalid sessions due to fully logging in/out too quickly. - This works by performing one full login as part of the setup of the test suite, and performing session logins for each test. - Login/Logout code is exercised as part of the setup/teardown of the test suite. - A `session.txt` file with the session string is created as part of the setup and removed on teardown. - `TestEventNotify` is skipped for now. --- mega_test.go | 300 ++++++++++++++++++++++++++++++--------------------- 1 file changed, 179 insertions(+), 121 deletions(-) diff --git a/mega_test.go b/mega_test.go index 1c3ca04..ef82557 100644 --- a/mega_test.go +++ b/mega_test.go @@ -44,6 +44,26 @@ func retry(t *testing.T, what string, fn func() error) { t.Fatalf("%s failed: %v", what, err) } +func retrySetup(what string, fn func() error) { + const maxTries = 10 + var err error + sleep := 100 * time.Millisecond + for i := 1; i <= maxTries; i++ { + err = fn() + if err == nil { + return + } + if err != EAGAIN { + break + } + fmt.Printf("%s failed %d/%d - retrying after %v sleep", what, i, maxTries, sleep) + time.Sleep(sleep) + sleep *= 2 + } + fmt.Printf("%s failed: %v", what, err) + os.Exit(1) +} + type CredentialType int const ( @@ -84,59 +104,134 @@ func getCredentials() (string, string, error) { // getCredentialsOrSkip retrieves user credentials based on the specified CredentialType. // It supports both standard and MFA-enabled credentials. -// If the requested credentials are missing or invalid, the function skips the test with an appropriate message. -func getCredentialsOrSkip(t *testing.T, credentialsType CredentialType) (string, string, string) { +func getCredentialsOrSkip(credentialsType CredentialType) (string, string, string, error) { switch credentialsType { case Credentials: user, password, err := getCredentials() if err != nil { - t.Skipf("Skipping test due to credentials error: %v", err) + return "", "", "", fmt.Errorf("Skipping test due to credentials error: %v", err) } - return user, password, "" + return user, password, "", nil case MfaCredentials: user, password, mfa_code, err := getMfaCredentials() if err != nil { - t.Skipf("Skipping test due to credentials error: %v", err) + + return "", "", "", fmt.Errorf("Skipping test due to credentials error: %v", err) } - return user, password, mfa_code + return user, password, mfa_code, nil case AnyCredentials: user, password, err := getCredentials() if err != nil { - t.Logf("Trying with MFA credentials instead, getting other credentials failed with error: %v", err) + fmt.Printf("Trying with MFA credentials instead, getting other credentials failed with error: %v", err) } else { - return user, password, "" + return user, password, "", nil } user, password, mfa_code, err := getMfaCredentials() if err != nil { - t.Skipf("Skipping test due to credentials error: %v", err) + + return "", "", "", fmt.Errorf("Skipping test due to credentials error: %v", err) } - return user, password, mfa_code + return user, password, mfa_code, nil default: - t.Fatal("Invalid credentials type") + return "", "", "", fmt.Errorf("Invalid credentials type") } +} - t.Fatal("Unreachable!") - return "", "", "" +func trySessionLogin(m *Mega, mandatory bool) error { + if _, err := os.Stat("session.txt"); err == nil { + session_file, err := os.ReadFile("session.txt") + if err != nil { + return fmt.Errorf("Error reading session file") + } else { + session := string(session_file) + err = m.SessionLogin(session) + if err != nil { + return fmt.Errorf("can't perform session login") + } + fmt.Printf("Using session: %s\n", session) + return nil + } + } + if (mandatory){ + return fmt.Errorf("Session file not found") + } + return nil } -func initSession(t *testing.T) *Mega { - user, password, mfa_code := getCredentialsOrSkip(t, AnyCredentials) +func setup() error { + m := New() - // m.SetDebugger(log.Printf) - retry(t, "Login", func() error { + + err := trySessionLogin(m, false) + if err != nil { + return err + } + + user, password, mfa_code, err := getCredentialsOrSkip(AnyCredentials) + if err != nil { + return fmt.Errorf("error getting credentials for login") + } + retrySetup("Login", func() error { if mfa_code != "" { return m.MultiFactorLogin(user, password, mfa_code) } return m.Login(user, password) }) - return m + + // dump session to file + session, err := m.DumpSession() + if err != nil { + return fmt.Errorf("Error dumping session") + } + fmt.Printf("Using session: %s\n", session) + err = os.WriteFile("session.txt", []byte(session), 0644) + if err != nil { + return fmt.Errorf("Error writing session file") + } + + return nil } -func endSession(t *testing.T, session *Mega) { - retry(t, "Logout", func() error { - return session.Logout() +func teardown() error{ + + fmt.Println("Logging out and removing session file.") + + m := New() + err := trySessionLogin(m, true) + if err != nil { + return err + } + + retrySetup("Logout", func() error { + return m.Logout() }) + _ = os.Remove("session.txt") + return nil +} + +func TestMain(m *testing.M) { + err:=setup() + if err != nil { + os.Exit(1) + } + exitCode := m.Run() + err=teardown() + if err != nil { + os.Exit(1) + } + os.Exit(exitCode) +} + +func initTest(t *testing.T) *Mega { + m := New() + + err := trySessionLogin(m, true) + if err != nil { + t.Fatal(err) + } + + return m } // createFile creates a temporary file of a given size along with its MD5SUM @@ -207,64 +302,36 @@ func fileMD5(t *testing.T, name string) string { return fmt.Sprintf("%x", h.Sum(nil)) } -func TestLogin(t *testing.T) { - user, password, _ := getCredentialsOrSkip(t, Credentials) - - m := New() - retry(t, "Login", func() error { - return m.Login(user, password) - }) - - endSession(t, m) -} - -func TestMfaLogin(t *testing.T) { - user, password, mfa_code := getCredentialsOrSkip(t, MfaCredentials) - - m := New() - retry(t, "MfaLogin", func() error { - return m.MultiFactorLogin(user, password, mfa_code) - }) - - endSession(t, m) -} - -func TestLogout(t *testing.T) { - session := initSession(t) - endSession(t, session) -} - func TestGetUser(t *testing.T) { - session := initSession(t) - _, err := session.GetUser() + m := initTest(t) + _, err := m.GetUser() if err != nil { t.Fatal("GetUser failed", err) } - endSession(t, session) } func TestUploadDownload(t *testing.T) { - session := initSession(t) + m := initTest(t) for i := range []int{0, 1} { if i == 0 { t.Log("HTTP Test") - session.SetHTTPS(false) + m.SetHTTPS(false) } else { t.Log("HTTPS Test") - session.SetHTTPS(true) + m.SetHTTPS(true) } - node, name, h1 := uploadFile(t, session, 314573, session.FS.root) + node, name, h1 := uploadFile(t, m, 314573, m.FS.root) - session.FS.mutex.Lock() - phash := session.FS.root.hash - n := session.FS.lookup[node.hash] + m.FS.mutex.Lock() + phash := m.FS.root.hash + n := m.FS.lookup[node.hash] if n.parent.hash != phash { t.Error("Parent of uploaded file mismatch") } - session.FS.mutex.Unlock() + m.FS.mutex.Unlock() - err := session.DownloadFile(node, name, nil) + err := m.DownloadFile(node, name, nil) if err != nil { t.Fatal("Download failed", err) } @@ -279,103 +346,96 @@ func TestUploadDownload(t *testing.T) { t.Error("MD5 mismatch for downloaded file") } } - session.SetHTTPS(false) - endSession(t, session) + m.SetHTTPS(false) } func TestMove(t *testing.T) { - session := initSession(t) - node, _, _ := uploadFile(t, session, 31, session.FS.root) + m := initTest(t) + node, _, _ := uploadFile(t, m, 31, m.FS.root) hash := node.hash - phash := session.FS.trash.hash - err := session.Move(node, session.FS.trash) + phash := m.FS.trash.hash + err := m.Move(node, m.FS.trash) if err != nil { t.Fatal("Move failed", err) } - session.FS.mutex.Lock() - n := session.FS.lookup[hash] + m.FS.mutex.Lock() + n := m.FS.lookup[hash] if n.parent.hash != phash { t.Error("Move happened to wrong parent", phash, n.parent.hash) } - session.FS.mutex.Unlock() - - endSession(t, session) + m.FS.mutex.Unlock() } func TestRename(t *testing.T) { - session := initSession(t) - node, _, _ := uploadFile(t, session, 31, session.FS.root) + m := initTest(t) + node, _, _ := uploadFile(t, m, 31, m.FS.root) - err := session.Rename(node, "newname.txt") + err := m.Rename(node, "newname.txt") if err != nil { t.Fatal("Rename failed", err) } - session.FS.mutex.Lock() - newname := session.FS.lookup[node.hash].name + m.FS.mutex.Lock() + newname := m.FS.lookup[node.hash].name if newname != "newname.txt" { t.Error("Renamed to wrong name", newname) } - session.FS.mutex.Unlock() - endSession(t, session) + m.FS.mutex.Unlock() } func TestDelete(t *testing.T) { - session := initSession(t) - node, _, _ := uploadFile(t, session, 31, session.FS.root) + m := initTest(t) + node, _, _ := uploadFile(t, m, 31, m.FS.root) retry(t, "Soft delete", func() error { - return session.Delete(node, false) + return m.Delete(node, false) }) - session.FS.mutex.Lock() - node = session.FS.lookup[node.hash] - if node.parent != session.FS.trash { + m.FS.mutex.Lock() + node = m.FS.lookup[node.hash] + if node.parent != m.FS.trash { t.Error("Expects file to be moved to trash") } - session.FS.mutex.Unlock() + m.FS.mutex.Unlock() retry(t, "Hard delete", func() error { - return session.Delete(node, true) + return m.Delete(node, true) }) time.Sleep(1 * time.Second) // wait for the event - session.FS.mutex.Lock() - if _, ok := session.FS.lookup[node.hash]; ok { + m.FS.mutex.Lock() + if _, ok := m.FS.lookup[node.hash]; ok { t.Error("Expects file to be dissapeared") } - session.FS.mutex.Unlock() - endSession(t, session) + m.FS.mutex.Unlock() } func TestGetUserSessions(t *testing.T) { - session := initSession(t) - _, err := session.GetUserSessions() + m := initTest(t) + _, err := m.GetUserSessions() if err != nil { t.Fatal("GetUserSessions failed", err) } - endSession(t, session) } func TestCreateDir(t *testing.T) { - session := initSession(t) - node := createDir(t, session, "testdir1", session.FS.root) - node2 := createDir(t, session, "testdir2", node) + m := initTest(t) + node := createDir(t, m, "testdir1", m.FS.root) + node2 := createDir(t, m, "testdir2", node) - session.FS.mutex.Lock() - nnode2 := session.FS.lookup[node2.hash] + m.FS.mutex.Lock() + nnode2 := m.FS.lookup[node2.hash] if nnode2.parent.hash != node.hash { t.Error("Wrong directory parent") } - session.FS.mutex.Unlock() - endSession(t, session) + m.FS.mutex.Unlock() } func TestConfig(t *testing.T) { - user, password, mfa_code := getCredentialsOrSkip(t, AnyCredentials) + user, password, mfa_code := "foo", "bar", "012345" m := New() m.SetAPIUrl("http://invalid.domain") @@ -405,22 +465,22 @@ func TestConfig(t *testing.T) { } func TestPathLookup(t *testing.T) { - session := initSession(t) + m := initTest(t) rs, err := randString(5) if err != nil { t.Fatalf("failed to make random string: %v", err) } - node1 := createDir(t, session, "dir-1-"+rs, session.FS.root) - node21 := createDir(t, session, "dir-2-1-"+rs, node1) - node22 := createDir(t, session, "dir-2-2-"+rs, node1) - node31 := createDir(t, session, "dir-3-1-"+rs, node21) - node32 := createDir(t, session, "dir-3-2-"+rs, node22) + node1 := createDir(t, m, "dir-1-"+rs, m.FS.root) + node21 := createDir(t, m, "dir-2-1-"+rs, node1) + node22 := createDir(t, m, "dir-2-2-"+rs, node1) + node31 := createDir(t, m, "dir-3-1-"+rs, node21) + node32 := createDir(t, m, "dir-3-2-"+rs, node22) _ = node32 - _, name1, _ := uploadFile(t, session, 31, node31) - _, _, _ = uploadFile(t, session, 31, node31) - _, name3, _ := uploadFile(t, session, 31, node22) + _, name1, _ := uploadFile(t, m, 31, node31) + _, _, _ = uploadFile(t, m, 31, node31) + _, name3, _ := uploadFile(t, m, 31, node22) testpaths := [][]string{ {"dir-1-" + rs, "dir-2-2-" + rs, path.Base(name3)}, @@ -432,7 +492,7 @@ func TestPathLookup(t *testing.T) { results := []error{nil, nil, nil, ENOENT} for i, tst := range testpaths { - ns, e := session.FS.PathLookup(session.FS.root, tst) + ns, e := m.FS.PathLookup(m.FS.root, tst) switch { case e != results[i]: t.Errorf("Test %d failed: wrong result", i) @@ -452,12 +512,13 @@ func TestPathLookup(t *testing.T) { } } } - endSession(t, session) } func TestEventNotify(t *testing.T) { - session1 := initSession(t) - session2 := initSession(t) + t.Skipf("TODO: reimplement this test") + + session1 := initTest(t) + session2 := initTest(t) node, _, _ := uploadFile(t, session1, 31, session1.FS.root) @@ -482,26 +543,23 @@ func TestEventNotify(t *testing.T) { if node != nil { t.Fatal("Expects file to not-found in first client's FS") } - endSession(t, session1) - endSession(t, session2) } func TestExportLink(t *testing.T) { - session := initSession(t) - node, _, _ := uploadFile(t, session, 31, session.FS.root) + m := initTest(t) + node, _, _ := uploadFile(t, m, 31, m.FS.root) // Don't include decryption key retry(t, "Failed to export link (key not included)", func() error { - _, err := session.Link(node, false) + _, err := m.Link(node, false) return err }) // Do include decryption key retry(t, "Failed to export link (key included)", func() error { - _, err := session.Link(node, true) + _, err := m.Link(node, true) return err }) - endSession(t, session) } func TestWaitEvents(t *testing.T) { From c5df93b8758bc79e1b7b6c9ee27c4475db0f6706 Mon Sep 17 00:00:00 2001 From: Francisco Pombal Date: Fri, 20 Sep 2024 10:57:49 +0100 Subject: [PATCH 10/10] Update README to reflect recent changes --- README.md | 79 ++++++++++++++++++++++++------------------------------- 1 file changed, 35 insertions(+), 44 deletions(-) diff --git a/README.md b/README.md index fb8161d..695fbd8 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,4 @@ -go-mega -======= +# go-mega A client library in go for mega.co.nz storage service. @@ -7,19 +6,22 @@ An implementation of command-line utility can be found at [https://github.com/t3 [![Build Status](https://secure.travis-ci.org/t3rm1n4l/go-mega.png?branch=master)](http://travis-ci.org/t3rm1n4l/go-mega) -### What can i do with this library? +## What can i do with this library? + This is an API client library for MEGA storage service. Currently, the library supports the basic APIs and operations as follows: - - User login - - Fetch filesystem tree - - Upload file - - Download file - - Create directory - - Move file or directory - - Rename file or directory - - Delete file or directory - - Parallel split download and upload - - Filesystem events auto sync - - Unit tests + +- User login with email and password, with or without 2FA/MFA +- User login with session token +- Fetch filesystem tree +- Upload file +- Download file +- Create directory +- Move file or directory +- Rename file or directory +- Delete file or directory +- Parallel split download and upload +- Filesystem events auto sync +- Unit tests ### API methods @@ -27,38 +29,27 @@ Please find full doc at [https://pkg.go.dev/github.com/t3rm1n4l/go-mega](https:/ ### Testing - export MEGA_USER= - export MEGA_PASSWD= - $ make test - go test -v - === RUN TestLogin - --- PASS: TestLogin (1.90 seconds) - === RUN TestGetUser - --- PASS: TestGetUser (1.65 seconds) - === RUN TestUploadDownload - --- PASS: TestUploadDownload (12.28 seconds) - === RUN TestMove - --- PASS: TestMove (9.31 seconds) - === RUN TestRename - --- PASS: TestRename (9.16 seconds) - === RUN TestDelete - --- PASS: TestDelete (3.87 seconds) - === RUN TestCreateDir - --- PASS: TestCreateDir (2.34 seconds) - === RUN TestConfig - --- PASS: TestConfig (0.01 seconds) - === RUN TestPathLookup - --- PASS: TestPathLookup (8.54 seconds) - === RUN TestEventNotify - --- PASS: TestEventNotify (19.65 seconds) - PASS - ok github.com/t3rm1n4l/go-mega68.745s +1. Export `MEGA_USER` and `MEGA_PASSWD` _or_ `MEGA_USER_MFA`, `MEGA_PASSWD_MFA` and `MEGA_SECRET_MFA` environment variables with your MEGA account credentials. +2. Run `go test -timeout 10m`. + + ```sh + $ MEGA_USER_MFA=user@email.com + $ MEGA_PASSWD_MFA="password" + $ MEGA_SECRET_MFA="${BASE64_ENCODED_MFA_SECRET}" + $ go test -timeout 10m + ... + ok github.com/t3rm1n4l/go-mega 36.772s + ``` + +A `session.txt` file will be created in the current directory with the session token during the tests. +It should be cleaned up after testing. ### TODO - - Implement APIs for public download url generation - - Implement download from public url - - Add shared user content management APIs - - Add contact list management APIs + +- Implement APIs for public download url generation +- Implement download from public url +- Add shared user content management APIs +- Add contact list management APIs ### License