From b8f9664905bac23ece2484b0466dd6ca41be5a05 Mon Sep 17 00:00:00 2001 From: Andrenerd Date: Sat, 17 Apr 2021 05:00:00 +0200 Subject: [PATCH 1/4] formatting --- code/go/0chain.net/core/go.sum | 5 ++ code/go/0chain.net/core/transaction/http.go | 82 +++++++++++++-------- 2 files changed, 55 insertions(+), 32 deletions(-) diff --git a/code/go/0chain.net/core/go.sum b/code/go/0chain.net/core/go.sum index cf8049351..1d5a44296 100644 --- a/code/go/0chain.net/core/go.sum +++ b/code/go/0chain.net/core/go.sum @@ -2,6 +2,7 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT github.com/0chain/gosdk v1.0.32/go.mod h1:TF9Yf4Z2dwYvAoqRACWK4rApeMFV+llAnYGlXTq7wjY= github.com/0chain/gosdk v1.0.40 h1:EDjJz0Q0bOQpkdnBG2qrnl3ym+qcB0tXFiQGoIH4kY8= github.com/0chain/gosdk v1.0.40/go.mod h1:TF9Yf4Z2dwYvAoqRACWK4rApeMFV+llAnYGlXTq7wjY= +github.com/0chain/gosdk v1.0.85 h1:ynXxR45bleZl/qV9u9kJDKTQkwUSwfL9J0fSTuSiDyw= github.com/0chain/gosdk v1.0.85/go.mod h1:edV5GwogiT6nK2+s4QcFCz3saUhkuFK6EIqNJfOt8xc= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= @@ -134,13 +135,16 @@ go.dedis.ch/protobuf v1.0.7/go.mod h1:pv5ysfkDX/EawiPqcW3ikOxsL5t+BqnV6xHSmE79KI go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.5.0 h1:KCa4XfM8CWFCpxXRGok+Q0SS/0XBhMDbHHGABQLvD2A= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.15.0 h1:ZZCA22JRF2gQE5FoNmhmrf7jeJJ2uhqDUNRYKm8dvmM= go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190123085648-057139ce5d2b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -148,6 +152,7 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8 h1:1wopBVtVdWnn03fZelqdXTqk7U7zPQCb+T4rbU9ZEoU= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56/go.mod h1:JhuoJpWY28nO4Vef9tZUw9qufEGTyX1+7lmHxV5q5G4= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= diff --git a/code/go/0chain.net/core/transaction/http.go b/code/go/0chain.net/core/transaction/http.go index 295e16536..41878a33b 100644 --- a/code/go/0chain.net/core/transaction/http.go +++ b/code/go/0chain.net/core/transaction/http.go @@ -127,11 +127,11 @@ func VerifyTransaction(txnHash string, chain *chain.Chain) (*Transaction, error) // Logger.Error("Error getting transaction confirmation", zap.Any("error", err)) // numSharders-- // } else { - // if response.StatusCode != 200 { + // if res.StatusCode != 200 { // continue // } - // defer response.Body.Close() - // contents, err := ioutil.ReadAll(response.Body) + // defer res.Body.Close() + // contents, err := ioutil.ReadAll(res.Body) // if err != nil { // Logger.Error("Error reading response from transaction confirmation", zap.Any("error", err)) // continue @@ -173,22 +173,27 @@ func VerifyTransaction(txnHash string, chain *chain.Chain) (*Transaction, error) } func MakeSCRestAPICall(scAddress string, relativePath string, params map[string]string, chain *chain.Chain, handler SCRestAPIHandler) ([]byte, error) { + var resMaxCounterBody []byte + resBodies := make(map[string][]byte) + + var hashMaxCounter int + hashCounters := make(map[string]int) + network := zcncore.GetNetwork() numSharders := len(network.Sharders) sharders := util.GetRandom(network.Sharders, numSharders) - responses := make(map[string]int) - entityResult := make(map[string][]byte) - var retObj []byte - maxCount := 0 + for _, sharder := range sharders { + hash := sha1.New() + urlString := fmt.Sprintf("%v/%v%v%v", sharder, SC_REST_API_URL, scAddress, relativePath) - urlObj, _ := url.Parse(urlString) - q := urlObj.Query() + url, _ := url.Parse(urlString) + q := url.Query() for k, v := range params { q.Add(k, v) } - urlObj.RawQuery = q.Encode() - h := sha1.New() + url.RawQuery = q.Encode() + var netTransport = &http.Transport{ Dial: (&net.Dialer{ Timeout: 5 * time.Second, @@ -199,46 +204,59 @@ func MakeSCRestAPICall(scAddress string, relativePath string, params map[string] Timeout: time.Second * 10, Transport: netTransport, } - response, err := netClient.Get(urlObj.String()) + + res, err := netClient.Get(url.String()) + if err != nil { Logger.Error("Error getting response for sc rest api", zap.Any("error", err), zap.Any("sharder_url", sharder)) numSharders-- } else { - if response.StatusCode != 200 { - responseBody, _ := ioutil.ReadAll(response.Body) - Logger.Error("Got error response from sc rest api", zap.Any("response", string(responseBody))) - response.Body.Close() + if res.StatusCode != 200 { + resBody, _ := ioutil.ReadAll(res.Body) + Logger.Error("Got error response from sc rest api", zap.Any("response", string(resBody))) + res.Body.Close() continue } - defer response.Body.Close() - tReader := io.TeeReader(response.Body, h) - entityBytes, err := ioutil.ReadAll(tReader) + + defer res.Body.Close() // TODO: is it really needed here? or put it above and drop other "Body.Close"s + + teeReader := io.TeeReader(res.Body, hash) + resBody, err := ioutil.ReadAll(teeReader) + if err != nil { Logger.Error("Error reading response", zap.Any("error", err)) - response.Body.Close() + res.Body.Close() continue } - hashBytes := h.Sum(nil) - hash := hex.EncodeToString(hashBytes) - responses[hash]++ - if responses[hash] > maxCount { - maxCount = responses[hash] - retObj = entityBytes + + hashString := hex.EncodeToString(hash.Sum(nil)) + hashCounters[hashString]++ + + if hashCounters[hashString] > hashMaxCounter { + hashMaxCounter = hashCounters[hashString] + resMaxCounterBody = resBody } - entityResult[sharder] = retObj - response.Body.Close() + + resBodies[sharder] = resMaxCounterBody // TODO: check it! looks suspicious. assigned value is not set for some interations. maybe should be = resBody? + res.Body.Close() } } + var err error - if maxCount <= (numSharders / 2) { + // is it less than or equal to 50% + if hashMaxCounter <= (numSharders / 2) { err = common.NewError("invalid_response", "Sharder responses were invalid. Hash mismatch") } + if handler != nil { - handler(entityResult, numSharders, err) + handler(resBodies, numSharders, err) } - if maxCount > (numSharders / 2) { - return retObj, nil + + // is it more than 50% + if hashMaxCounter > (numSharders / 2) { + return resMaxCounterBody, nil } + return nil, err } From 560503b719e09da07e97e5d25f3bdd996a0d6c55 Mon Sep 17 00:00:00 2001 From: Andrenerd Date: Sat, 17 Apr 2021 05:23:07 +0200 Subject: [PATCH 2/4] make extra calls to temporarily unavailable sharders --- code/go/0chain.net/core/transaction/http.go | 29 ++++++++++++++++----- 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/code/go/0chain.net/core/transaction/http.go b/code/go/0chain.net/core/transaction/http.go index 41878a33b..bb0851c1c 100644 --- a/code/go/0chain.net/core/transaction/http.go +++ b/code/go/0chain.net/core/transaction/http.go @@ -194,18 +194,35 @@ func MakeSCRestAPICall(scAddress string, relativePath string, params map[string] } url.RawQuery = q.Encode() - var netTransport = &http.Transport{ + // Make one or more requests (in case of unavailability, see 503/504 errors) + var err error + var res *http.Response + var counter int = 3 // TODO: better create a const + + netTransport := &http.Transport{ Dial: (&net.Dialer{ - Timeout: 5 * time.Second, + Timeout: 5 * time.Second, // TODO: better create a const }).Dial, - TLSHandshakeTimeout: 5 * time.Second, + TLSHandshakeTimeout: 5 * time.Second, // TODO: better create a const } - var netClient = &http.Client{ - Timeout: time.Second * 10, + + netClient := &http.Client{ + Timeout: time.Second * 10, // TODO: create a const Transport: netTransport, } - res, err := netClient.Get(url.String()) + for counter > 0 { + res, err := netClient.Get(url.String()) + if err != nil { break } + + // TODO: better create an utility function as isAvailable or so + if (res.StatusCode == 503 || res.StatusCode == 504) { + res.Body.Close() + counter-- + } else { + break + } + } if err != nil { Logger.Error("Error getting response for sc rest api", zap.Any("error", err), zap.Any("sharder_url", sharder)) From a5ff0da01d0d6706f5d863e165ffdc338131cd43 Mon Sep 17 00:00:00 2001 From: Andrenerd Date: Sat, 17 Apr 2021 08:07:07 +0200 Subject: [PATCH 3/4] hotfix --- code/go/0chain.net/core/transaction/http.go | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/code/go/0chain.net/core/transaction/http.go b/code/go/0chain.net/core/transaction/http.go index bb0851c1c..eb9d0f6d0 100644 --- a/code/go/0chain.net/core/transaction/http.go +++ b/code/go/0chain.net/core/transaction/http.go @@ -184,16 +184,6 @@ func MakeSCRestAPICall(scAddress string, relativePath string, params map[string] sharders := util.GetRandom(network.Sharders, numSharders) for _, sharder := range sharders { - hash := sha1.New() - - urlString := fmt.Sprintf("%v/%v%v%v", sharder, SC_REST_API_URL, scAddress, relativePath) - url, _ := url.Parse(urlString) - q := url.Query() - for k, v := range params { - q.Add(k, v) - } - url.RawQuery = q.Encode() - // Make one or more requests (in case of unavailability, see 503/504 errors) var err error var res *http.Response @@ -211,8 +201,16 @@ func MakeSCRestAPICall(scAddress string, relativePath string, params map[string] Transport: netTransport, } + uString := fmt.Sprintf("%v/%v%v%v", sharder, SC_REST_API_URL, scAddress, relativePath) + u, _ := url.Parse(uString) + q := u.Query() + for k, v := range params { + q.Add(k, v) + } + u.RawQuery = q.Encode() + for counter > 0 { - res, err := netClient.Get(url.String()) + res, err := netClient.Get(u.String()) if err != nil { break } // TODO: better create an utility function as isAvailable or so @@ -237,6 +235,7 @@ func MakeSCRestAPICall(scAddress string, relativePath string, params map[string] defer res.Body.Close() // TODO: is it really needed here? or put it above and drop other "Body.Close"s + hash := sha1.New() teeReader := io.TeeReader(res.Body, hash) resBody, err := ioutil.ReadAll(teeReader) From 7139113129cff54b3a1d58b87336a7085cd39973 Mon Sep 17 00:00:00 2001 From: Andrenerd Date: Sun, 18 Apr 2021 22:30:27 +0200 Subject: [PATCH 4/4] formatting, fixes --- code/go/0chain.net/core/transaction/http.go | 45 +++++++++++---------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/code/go/0chain.net/core/transaction/http.go b/code/go/0chain.net/core/transaction/http.go index eb9d0f6d0..11bab7fb7 100644 --- a/code/go/0chain.net/core/transaction/http.go +++ b/code/go/0chain.net/core/transaction/http.go @@ -31,7 +31,10 @@ const TXN_VERIFY_URL = "v1/transaction/get/confirmation?hash=" const SC_REST_API_URL = "v1/screst/" const REGISTER_CLIENT = "v1/client/put" -const SLEEP_FOR_TXN_CONFIRMATION = 5 +const ( + SLEEP_FOR_TXN_CONFIRMATION = 5 + SC_REST_API_ATTEMPTS = 3 +) var ErrNoTxnDetail = common.NewError("missing_transaction_detail", "No transaction detail was found on any of the sharders") @@ -122,16 +125,16 @@ func VerifyTransaction(txnHash string, chain *chain.Chain) (*Transaction, error) // Timeout: time.Second * 10, // Transport: netTransport, // } - // response, err := netClient.Get(url) + // resp, err := netClient.Get(url) // if err != nil { // Logger.Error("Error getting transaction confirmation", zap.Any("error", err)) // numSharders-- // } else { - // if res.StatusCode != 200 { + // if resp.StatusCode != 200 { // continue // } - // defer res.Body.Close() - // contents, err := ioutil.ReadAll(res.Body) + // defer resp.Body.Close() + // contents, err := ioutil.ReadAll(resp.Body) // if err != nil { // Logger.Error("Error reading response from transaction confirmation", zap.Any("error", err)) // continue @@ -186,18 +189,18 @@ func MakeSCRestAPICall(scAddress string, relativePath string, params map[string] for _, sharder := range sharders { // Make one or more requests (in case of unavailability, see 503/504 errors) var err error - var res *http.Response - var counter int = 3 // TODO: better create a const + var resp *http.Response + var counter int = SC_REST_API_ATTEMPTS netTransport := &http.Transport{ Dial: (&net.Dialer{ - Timeout: 5 * time.Second, // TODO: better create a const + Timeout: 5 * time.Second, }).Dial, - TLSHandshakeTimeout: 5 * time.Second, // TODO: better create a const + TLSHandshakeTimeout: 5 * time.Second, } netClient := &http.Client{ - Timeout: time.Second * 10, // TODO: create a const + Timeout: 10 * time.Second, Transport: netTransport, } @@ -210,12 +213,12 @@ func MakeSCRestAPICall(scAddress string, relativePath string, params map[string] u.RawQuery = q.Encode() for counter > 0 { - res, err := netClient.Get(u.String()) + resp, err = netClient.Get(u.String()) if err != nil { break } - // TODO: better create an utility function as isAvailable or so - if (res.StatusCode == 503 || res.StatusCode == 504) { - res.Body.Close() + // if it's not available, retry if there are any retry attempts + if (resp.StatusCode == 503 || resp.StatusCode == 504) { + resp.Body.Close() counter-- } else { break @@ -226,22 +229,22 @@ func MakeSCRestAPICall(scAddress string, relativePath string, params map[string] Logger.Error("Error getting response for sc rest api", zap.Any("error", err), zap.Any("sharder_url", sharder)) numSharders-- } else { - if res.StatusCode != 200 { - resBody, _ := ioutil.ReadAll(res.Body) + if resp.StatusCode != 200 { + resBody, _ := ioutil.ReadAll(resp.Body) Logger.Error("Got error response from sc rest api", zap.Any("response", string(resBody))) - res.Body.Close() + resp.Body.Close() continue } - defer res.Body.Close() // TODO: is it really needed here? or put it above and drop other "Body.Close"s + defer resp.Body.Close() // TODO: is it really needed here? or put it above and drop other "Body.Close"s hash := sha1.New() - teeReader := io.TeeReader(res.Body, hash) + teeReader := io.TeeReader(resp.Body, hash) resBody, err := ioutil.ReadAll(teeReader) if err != nil { Logger.Error("Error reading response", zap.Any("error", err)) - res.Body.Close() + resp.Body.Close() continue } @@ -254,7 +257,7 @@ func MakeSCRestAPICall(scAddress string, relativePath string, params map[string] } resBodies[sharder] = resMaxCounterBody // TODO: check it! looks suspicious. assigned value is not set for some interations. maybe should be = resBody? - res.Body.Close() + resp.Body.Close() } }