Skip to content

Commit

Permalink
client: remove request body buffering
Browse files Browse the repository at this point in the history 
This should prevent any request bodies from being buffered in memory in
their entirety before being sent.

Signed-off-by: Hank Donnay <hdonnay@redhat.com>
  • Loading branch information
@hdonnay
hdonnay committed Mar 10, 2021
1 parent 1ac26da commit bd50a95
Show file tree
Hide file tree
Showing 2 changed files with 92 additions and 54 deletions.
110 changes: 69 additions & 41 deletions httptransport/client/indexer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"path"

Expand All @@ -18,38 +19,50 @@ import (
var _ indexer.Service = (*HTTP)(nil)

func (s *HTTP) AffectedManifests(ctx context.Context, v []claircore.Vulnerability) (*claircore.AffectedManifests, error) {
var affected claircore.AffectedManifests
buf := bytes.NewBuffer([]byte{})
err := json.NewEncoder(buf).Encode(struct {
V []claircore.Vulnerability `json:"vulnerabilities"`
}{
v,
})
if err != nil {
return nil, &clairerror.ErrBadVulnerabilities{err}
}

u, err := s.addr.Parse(httptransport.AffectedManifestAPIPath)
if err != nil {
return nil, fmt.Errorf("failed to parse api address: %v", err)
}
req, err := http.NewRequestWithContext(ctx, "POST", u.String(), buf)
rd, wr := io.Pipe()
go func() {
defer wr.Close()
if err := json.NewEncoder(wr).Encode(struct {
V []claircore.Vulnerability `json:"vulnerabilities"`
}{
v,
}); err != nil {
wr.CloseWithError(err)
}
}()
defer rd.Close()

req, err := http.NewRequestWithContext(ctx, http.MethodPost, u.String(), rd)
if err != nil {
return nil, fmt.Errorf("failed to create request: %v", err)
}
req.Header.Set("content-type", `application/json`)
resp, err := s.c.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, &clairerror.ErrRequestFail{Code: resp.StatusCode, Status: resp.Status}
return nil, &clairerror.ErrRequestFail{
Code: resp.StatusCode,
Status: resp.Status,
}
}
err = json.NewDecoder(resp.Body).Decode(&affected)
if err != nil {
return nil, &clairerror.ErrBadAffectedManifests{err}
}
return &affected, nil

var a claircore.AffectedManifests
switch ct := req.Header.Get("content-type"); ct {
case "", `application/json`:
if err := json.NewDecoder(resp.Body).Decode(&a); err != nil {
return nil, err
}
default:
return nil, fmt.Errorf("unrecognized content-type %q", ct)
}
return &a, nil
}

// Index receives a Manifest and returns a IndexReport providing the indexed
Expand All @@ -59,36 +72,46 @@ func (s *HTTP) AffectedManifests(ctx context.Context, v []claircore.Vulnerabilit
// could not start. If an error occurs during the index operation the error will
// be preset on the IndexReport.Err field of the returned IndexReport.
func (s *HTTP) Index(ctx context.Context, manifest *claircore.Manifest) (*claircore.IndexReport, error) {
buf := bytes.NewBuffer([]byte{})
err := json.NewEncoder(buf).Encode(manifest)
if err != nil {
return nil, &clairerror.ErrBadManifest{err}
}

u, err := s.addr.Parse(httptransport.IndexAPIPath)
if err != nil {
return nil, fmt.Errorf("failed to create request: %v", err)
}
req, err := http.NewRequestWithContext(ctx, "POST", u.String(), buf)
rd, wr := io.Pipe()
go func() {
defer wr.Close()
if err := json.NewEncoder(wr).Encode(manifest); err != nil {
wr.CloseWithError(err)
}
}()
defer rd.Close()

req, err := http.NewRequestWithContext(ctx, http.MethodPost, u.String(), rd)
if err != nil {
return nil, fmt.Errorf("failed to create request: %v", err)
}
req.Header.Set("content-type", `application/json`)
resp, err := s.c.Do(req)
if err != nil {
return nil, fmt.Errorf("failed to do request: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, &clairerror.ErrRequestFail{Code: resp.StatusCode, Status: resp.Status}
}

var sr *claircore.IndexReport
err = json.NewDecoder(resp.Body).Decode(sr)
if err != nil {
return nil, &clairerror.ErrBadIndexReport{err}
return nil, &clairerror.ErrRequestFail{
Code: resp.StatusCode,
Status: resp.Status,
}
}

return sr, nil
var ir claircore.IndexReport
switch ct := resp.Header.Get("content-type"); ct {
case "", `application/json`:
if err := json.NewDecoder(resp.Body).Decode(&ir); err != nil {
return nil, err
}
default:
return nil, fmt.Errorf("unrecognized content-type %q", ct)
}
return &ir, nil
}

// IndexReport retrieves a IndexReport given a manifest hash string
Expand All @@ -97,7 +120,8 @@ func (s *HTTP) IndexReport(ctx context.Context, manifest claircore.Digest) (*cla
if err != nil {
return nil, false, fmt.Errorf("failed to create request: %v", err)
}
req, err := http.NewRequestWithContext(ctx, "GET", u.String(), nil)

req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
if err != nil {
return nil, false, fmt.Errorf("failed to create request: %v", err)
}
Expand All @@ -106,19 +130,23 @@ func (s *HTTP) IndexReport(ctx context.Context, manifest claircore.Digest) (*cla
return nil, false, fmt.Errorf("failed to do request: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusNotFound {
switch resp.StatusCode {
case http.StatusOK:
case http.StatusNotFound:
return nil, false, nil
}
if resp.StatusCode != http.StatusOK {
return nil, false, &clairerror.ErrIndexReportRetrieval{&clairerror.ErrRequestFail{Code: resp.StatusCode, Status: resp.Status}}
default:
return nil, false, &clairerror.ErrIndexReportRetrieval{
E: &clairerror.ErrRequestFail{
Code: resp.StatusCode,
Status: resp.Status,
}}
}

ir := &claircore.IndexReport{}
err = json.NewDecoder(resp.Body).Decode(ir)
if err != nil {
return nil, false, &clairerror.ErrBadIndexReport{err}
return nil, false, &clairerror.ErrBadIndexReport{E: err}
}

return ir, true, nil
}

Expand All @@ -127,7 +155,7 @@ func (s *HTTP) State(ctx context.Context) (string, error) {
if err != nil {
return "", fmt.Errorf("failed to create request: %v", err)
}
req, err := http.NewRequestWithContext(ctx, "GET", u.String(), nil)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
if err != nil {
return "", fmt.Errorf("failed to create request: %v", err)
}
Expand Down
36 changes: 23 additions & 13 deletions httptransport/client/matcher.go
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
package client

import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"strings"
"sync"
Expand All @@ -14,6 +14,7 @@ import (
"github.com/quay/claircore"
"github.com/quay/claircore/libvuln/driver"

clairerror "github.com/quay/clair/v4/clair-error"
"github.com/quay/clair/v4/httptransport"
"github.com/quay/clair/v4/matcher"
)
Expand All @@ -25,31 +26,40 @@ func (c *HTTP) Scan(ctx context.Context, ir *claircore.IndexReport) (*claircore.
if err != nil {
return nil, err
}
rd, wr := io.Pipe()
go func() {
defer wr.Close()
if err := json.NewEncoder(wr).Encode(ir); err != nil {
wr.CloseWithError(err)
}
}()
defer rd.Close()

b, err := json.Marshal(&ir)
if err != nil {
return nil, err
}
buf := bytes.NewBuffer(b)

req, err := http.NewRequestWithContext(ctx, http.MethodPost, u.String(), buf)
req, err := http.NewRequestWithContext(ctx, http.MethodPost, u.String(), rd)
if err != nil {
return nil, err
}

req.Header.Set("content-type", `application/json`)
resp, err := c.c.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("%v: unexpected status: %s", u.Path, resp.Status)
return nil, &clairerror.ErrRequestFail{
Code: resp.StatusCode,
Status: resp.Status,
}
}

var vr claircore.VulnerabilityReport
err = json.NewDecoder(resp.Body).Decode(&vr)
if err != nil {
return nil, err
switch ct := req.Header.Get("content-type"); ct {
case "", `application/json`:
if err := json.NewDecoder(resp.Body).Decode(&vr); err != nil {
return nil, err
}
default:
return nil, fmt.Errorf("unrecognized content-type %q", ct)
}
return &vr, nil
}
Expand Down

0 comments on commit bd50a95

Please sign in to comment.