This repository has been archived by the owner on Jan 10, 2023. It is now read-only.
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Wait For Titus Isolate to Isolate Workload Before Starting Workload
Titus isolate waits for Docker events, and such. This sometimes can take a moment. In order to avoid racing, and have the application read the CPU state, and then configure itself based on that CPU state, we wait for titus-isolate up to some timeout.
- Loading branch information
Showing
3 changed files
with
186 additions
and
22 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,58 @@ | ||
| package docker | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "net/http" | ||
| "net/url" | ||
| "time" | ||
|
|
||
| "github.com/sirupsen/logrus" | ||
| ) | ||
|
|
||
| const ( | ||
| titusIsolateHost = "localhost:7500" | ||
| ) | ||
|
|
||
| func waitForTitusIsolate(parentCtx context.Context, taskID string, timeout time.Duration) { | ||
| waitForTitusIsolateWithHost(parentCtx, taskID, titusIsolateHost, timeout) | ||
| } | ||
|
|
||
| func waitForTitusIsolateWithHost(parentCtx context.Context, taskID, host string, timeout time.Duration) { | ||
| ctx, cancel := context.WithTimeout(parentCtx, timeout) | ||
| defer cancel() | ||
| requestURL := &url.URL{ | ||
| Scheme: "http", | ||
| Host: host, | ||
| Path: fmt.Sprintf("/isolate/%s", taskID), | ||
| } | ||
|
|
||
| rq, err := http.NewRequest("GET", requestURL.String(), http.NoBody) | ||
| if err != nil { | ||
| logrus.WithError(err).Warn("Could not form HTTP Request to Isolate") | ||
| return | ||
| } | ||
| rq = rq.WithContext(ctx) | ||
| client := &http.Client{ | ||
| Transport: &http.Transport{ | ||
| DisableKeepAlives: true, | ||
| }, | ||
| } | ||
| resp, err := client.Do(rq) | ||
| // Check if context timed out | ||
| if err != nil { | ||
| logrus.WithError(err).Warn("Error calling titus isolate") | ||
| return | ||
| } | ||
|
|
||
| defer shouldClose(resp.Body) | ||
|
|
||
| if resp.StatusCode == 200 { | ||
| return | ||
| } | ||
|
|
||
| logrus.WithFields(map[string]interface{}{ | ||
| "statusCode": resp.StatusCode, | ||
| "status": resp.Status, | ||
| }).Warn("Titus Isolate did not return code 200") | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,65 @@ | ||
| package docker | ||
|
|
||
| import ( | ||
| "context" | ||
| "net/http" | ||
| "net/http/httptest" | ||
| "testing" | ||
| "time" | ||
|
|
||
| "github.com/stretchr/testify/assert" | ||
| ) | ||
|
|
||
| func TestTitusIsolateTimeout(t *testing.T) { | ||
| t.Parallel() | ||
| handler := func(w http.ResponseWriter, r *http.Request) { | ||
| // This function will not return quickly. | ||
| t := time.NewTimer(30 * time.Second) | ||
| defer t.Stop() | ||
| select { | ||
| case <-t.C: | ||
| case <-r.Context().Done(): | ||
| } | ||
| w.WriteHeader(200) | ||
| } | ||
| server := httptest.NewServer(http.HandlerFunc(handler)) | ||
| defer server.Close() | ||
|
|
||
| ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) | ||
| defer cancel() | ||
| now := time.Now() | ||
| waitForTitusIsolateWithHost(ctx, "foo", server.Listener.Addr().String(), 1*time.Second) | ||
| assert.True(t, time.Since(now) < 15*time.Second) | ||
| } | ||
|
|
||
| func TestTitusIsolateSuccess(t *testing.T) { | ||
| t.Parallel() | ||
| handler := func(w http.ResponseWriter, r *http.Request) { | ||
| // This function will return quickly. | ||
| w.WriteHeader(200) | ||
| } | ||
| server := httptest.NewServer(http.HandlerFunc(handler)) | ||
| defer server.Close() | ||
|
|
||
| ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) | ||
| defer cancel() | ||
| now := time.Now() | ||
| waitForTitusIsolateWithHost(ctx, "foo", server.Listener.Addr().String(), 1*time.Second) | ||
| assert.True(t, time.Since(now) < 5*time.Second) | ||
| } | ||
|
|
||
| func TestTitusIsolate404(t *testing.T) { | ||
| t.Parallel() | ||
| handler := func(w http.ResponseWriter, r *http.Request) { | ||
| // This function will return quickly. | ||
| w.WriteHeader(404) | ||
| } | ||
| server := httptest.NewServer(http.HandlerFunc(handler)) | ||
| defer server.Close() | ||
|
|
||
| ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) | ||
| defer cancel() | ||
| now := time.Now() | ||
| waitForTitusIsolateWithHost(ctx, "foo", server.Listener.Addr().String(), 1*time.Second) | ||
| assert.True(t, time.Since(now) < 5*time.Second) | ||
| } |