Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Browse files
Browse the repository at this point in the history
- Loading branch information
Showing
6 changed files
with
281 additions
and
8 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,100 @@ | ||
| package nw_simulator | ||
|
|
||
| import ( | ||
| "bytes" | ||
| "fmt" | ||
| "github.com/watermint/toolbox/essentials/http/es_response" | ||
| "github.com/watermint/toolbox/essentials/http/es_response_impl" | ||
| "github.com/watermint/toolbox/essentials/network/nw_client" | ||
| "github.com/watermint/toolbox/essentials/network/nw_concurrency" | ||
| "github.com/watermint/toolbox/essentials/network/nw_ratelimit" | ||
| "github.com/watermint/toolbox/essentials/network/nw_retry" | ||
| "github.com/watermint/toolbox/infra/api/api_context" | ||
| "io/ioutil" | ||
| "math/rand" | ||
| "net/http" | ||
| "time" | ||
| ) | ||
|
|
||
| const ( | ||
| maxRetryAfter = 4 | ||
|
|
||
| RetryAfterHeaderRetryAfter = iota | ||
| RetryAfterHeaderGitHub | ||
| RetryAfterHeaderIetfDraftSecond | ||
| RetryAfterHeaderIetfDraftTimestamp | ||
| ) | ||
|
|
||
| type RetryAfterHeaderType int | ||
| type ResponseDecorator func(res *http.Response) | ||
|
|
||
| func NoDecorator(res *http.Response) { | ||
| } | ||
|
|
||
| func New(client nw_client.Rest, rate int, headerType RetryAfterHeaderType, decorator ResponseDecorator) nw_client.Rest { | ||
| return &narrowClient{ | ||
| rate: rate, | ||
| headerType: headerType, | ||
| decorator: decorator, | ||
| client: client, | ||
| } | ||
| } | ||
|
|
||
| type narrowClient struct { | ||
| // too many requests error rate in percent | ||
| rate int | ||
|
|
||
| // retry after header type | ||
| headerType RetryAfterHeaderType | ||
|
|
||
| // response decorator | ||
| decorator ResponseDecorator | ||
|
|
||
| // nested client | ||
| client nw_client.Rest | ||
| } | ||
|
|
||
| func (z narrowClient) Call(ctx api_context.Context, req nw_client.RequestBuilder) (res es_response.Response) { | ||
| if rand.Intn(100) >= z.rate { | ||
| return z.client.Call(ctx, req) | ||
| } else { | ||
| hr := &http.Response{} | ||
| hr.StatusCode = http.StatusTooManyRequests | ||
| hr.Header = make(map[string][]string) | ||
|
|
||
| retryAfterSec := rand.Intn(maxRetryAfter) + 1 | ||
|
|
||
| switch z.headerType { | ||
| case RetryAfterHeaderGitHub: | ||
| hr.Header.Add(nw_retry.HeaderXRateLimitLimit, "100") | ||
| hr.Header.Add(nw_retry.HeaderXRateLimitRemaining, "0") | ||
| hr.Header.Add(nw_retry.HeaderXRateLimitReset, fmt.Sprintf("%d", time.Now().Add(time.Duration(retryAfterSec)*time.Second).Unix())) | ||
|
|
||
| case RetryAfterHeaderIetfDraftTimestamp: | ||
| hr.Header.Add(nw_retry.HeaderRateLimitLimit, "100") | ||
| hr.Header.Add(nw_retry.HeaderRateLimitRemaining, "0") | ||
| hr.Header.Add(nw_retry.HeaderRateLimitReset, time.Now().Add(time.Duration(retryAfterSec)*time.Second).Format(time.RFC1123)) | ||
|
|
||
| case RetryAfterHeaderIetfDraftSecond: | ||
| hr.Header.Add(nw_retry.HeaderRateLimitLimit, "100") | ||
| hr.Header.Add(nw_retry.HeaderRateLimitRemaining, "0") | ||
| hr.Header.Add(nw_retry.HeaderRateLimitReset, fmt.Sprintf("%d", retryAfterSec)) | ||
|
|
||
| default: | ||
| hr.Header.Add(nw_retry.HeaderRetryAfter, fmt.Sprintf("%d", retryAfterSec)) | ||
| } | ||
|
|
||
| if z.decorator != nil { | ||
| z.decorator(hr) | ||
| } | ||
| if hr.Body == nil { | ||
| hr.Body = ioutil.NopCloser(&bytes.Buffer{}) | ||
| } | ||
|
|
||
| nw_ratelimit.WaitIfRequired(ctx.ClientHash(), req.Endpoint()) | ||
| nw_concurrency.Start() | ||
| res := es_response_impl.New(ctx, hr) | ||
| nw_concurrency.End() | ||
| return res | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,115 @@ | ||
| package nw_simulator | ||
|
|
||
| import ( | ||
| "bytes" | ||
| "github.com/watermint/toolbox/essentials/http/es_response" | ||
| "github.com/watermint/toolbox/essentials/log/esl" | ||
| "github.com/watermint/toolbox/essentials/network/nw_client" | ||
| "github.com/watermint/toolbox/essentials/network/nw_retry" | ||
| "github.com/watermint/toolbox/infra/api/api_context" | ||
| "net/http" | ||
| "strconv" | ||
| "testing" | ||
| "time" | ||
| ) | ||
|
|
||
| type PanicClient struct { | ||
| } | ||
|
|
||
| func (p PanicClient) Call(ctx api_context.Context, req nw_client.RequestBuilder) (res es_response.Response) { | ||
| panic("always panic!") | ||
| } | ||
|
|
||
| type MockApiContext struct { | ||
| } | ||
|
|
||
| func (z MockApiContext) ClientHash() string { | ||
| return "" | ||
| } | ||
|
|
||
| func (z MockApiContext) Log() esl.Logger { | ||
| return esl.Default() | ||
| } | ||
|
|
||
| func (z MockApiContext) Capture() esl.Logger { | ||
| return esl.Default() | ||
| } | ||
|
|
||
| type MockReqBuilder struct { | ||
| } | ||
|
|
||
| func (z MockReqBuilder) Build() (*http.Request, error) { | ||
| return http.NewRequest("POST", z.Endpoint(), &bytes.Buffer{}) | ||
| } | ||
|
|
||
| func (z MockReqBuilder) Endpoint() string { | ||
| return "http://www.example.com" | ||
| } | ||
|
|
||
| func (z MockReqBuilder) Param() string { | ||
| return "" | ||
| } | ||
|
|
||
| func TestNarrowClient_Call(t *testing.T) { | ||
| { | ||
| nc := New(&PanicClient{}, 100, RetryAfterHeaderRetryAfter, NoDecorator) | ||
| res := nc.Call(&MockApiContext{}, &MockReqBuilder{}) | ||
| if res.IsSuccess() { | ||
| t.Error(res.IsSuccess()) | ||
| } | ||
| if res.Code() != http.StatusTooManyRequests { | ||
| t.Error(res.Code()) | ||
| } | ||
| v := res.Header(nw_retry.HeaderRetryAfter) | ||
| if va, err := strconv.Atoi(v); err != nil || va < 1 { | ||
| t.Error(err, va) | ||
| } | ||
| } | ||
|
|
||
| { | ||
| nc := New(&PanicClient{}, 100, RetryAfterHeaderGitHub, NoDecorator) | ||
| now := time.Now().Unix() | ||
| res := nc.Call(&MockApiContext{}, &MockReqBuilder{}) | ||
| if res.IsSuccess() { | ||
| t.Error(res.IsSuccess()) | ||
| } | ||
| if res.Code() != http.StatusTooManyRequests { | ||
| t.Error(res.Code()) | ||
| } | ||
| v := res.Header(nw_retry.HeaderXRateLimitReset) | ||
| if va, err := strconv.Atoi(v); err != nil || int64(va) < now { | ||
| t.Error(err, va) | ||
| } | ||
| } | ||
|
|
||
| { | ||
| nc := New(&PanicClient{}, 100, RetryAfterHeaderIetfDraftTimestamp, NoDecorator) | ||
| now := time.Now() | ||
| res := nc.Call(&MockApiContext{}, &MockReqBuilder{}) | ||
| if res.IsSuccess() { | ||
| t.Error(res.IsSuccess()) | ||
| } | ||
| if res.Code() != http.StatusTooManyRequests { | ||
| t.Error(res.Code()) | ||
| } | ||
| v := res.Header(nw_retry.HeaderRateLimitReset) | ||
| if va, err := time.Parse(time.RFC1123, v); err != nil || va.Before(now) { | ||
| t.Error(err, va) | ||
| } | ||
| } | ||
|
|
||
| { | ||
| nc := New(&PanicClient{}, 100, RetryAfterHeaderIetfDraftSecond, NoDecorator) | ||
| res := nc.Call(&MockApiContext{}, &MockReqBuilder{}) | ||
| if res.IsSuccess() { | ||
| t.Error(res.IsSuccess()) | ||
| } | ||
| if res.Code() != http.StatusTooManyRequests { | ||
| t.Error(res.Code()) | ||
| } | ||
| v := res.Header(nw_retry.HeaderRateLimitReset) | ||
| if va, err := strconv.Atoi(v); err != nil || va < 1 { | ||
| t.Error(err, va) | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters