Skip to content
Permalink
Browse files

Merge pull request #74106 from caesarxuchao/automated-cherry-pick-of-…

…#68428-#68442-#73443-#73713-#73805-#74000-upstream-release-1.11

Automated cherry pick of #68428: vendor: bump github.com/evanphx/json-patch
  • Loading branch information...
k8s-ci-robot committed Feb 23, 2019
2 parents 8c4e545 + 8a2d338 commit 5d86c1bcf73cdd7d77d211693b43a90456a1f26e
Showing with 475 additions and 85 deletions.
  1. +2 −2 Godeps/Godeps.json
  2. +2 −0 cmd/kube-apiserver/app/options/options_test.go
  3. +1 −1 staging/src/k8s.io/apiextensions-apiserver/Godeps/Godeps.json
  4. +1 −1 staging/src/k8s.io/apimachinery/Godeps/Godeps.json
  5. +24 −0 staging/src/k8s.io/apimachinery/pkg/api/errors/errors.go
  6. +4 −0 staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/types.go
  7. +1 −1 staging/src/k8s.io/apiserver/Godeps/Godeps.json
  8. +4 −0 staging/src/k8s.io/apiserver/pkg/endpoints/groupversion.go
  9. +1 −1 staging/src/k8s.io/apiserver/pkg/endpoints/handlers/create.go
  10. +2 −2 staging/src/k8s.io/apiserver/pkg/endpoints/handlers/delete.go
  11. +17 −3 staging/src/k8s.io/apiserver/pkg/endpoints/handlers/patch.go
  12. +19 −2 staging/src/k8s.io/apiserver/pkg/endpoints/handlers/rest.go
  13. +1 −1 staging/src/k8s.io/apiserver/pkg/endpoints/handlers/update.go
  14. +2 −0 staging/src/k8s.io/apiserver/pkg/endpoints/installer.go
  15. +1 −0 staging/src/k8s.io/apiserver/pkg/server/BUILD
  16. +54 −15 staging/src/k8s.io/apiserver/pkg/server/config.go
  17. +5 −0 staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go
  18. +22 −2 staging/src/k8s.io/apiserver/pkg/server/options/server_run_options.go
  19. +1 −1 staging/src/k8s.io/kube-aggregator/Godeps/Godeps.json
  20. +1 −1 staging/src/k8s.io/sample-apiserver/Godeps/Godeps.json
  21. +3 −0 test/integration/apiserver/BUILD
  22. +69 −0 test/integration/apiserver/max_json_patch_operations_test.go
  23. +101 −0 test/integration/apiserver/max_request_body_bytes_test.go
  24. +0 −2 test/integration/examples/BUILD
  25. +2 −1 test/integration/examples/webhook_test.go
  26. +4 −0 test/integration/framework/BUILD
  27. +3 −4 test/integration/{examples/setup_test.go → framework/test_server.go}
  28. +1 −0 vendor/github.com/evanphx/json-patch/BUILD
  29. +14 −1 vendor/github.com/evanphx/json-patch/README.md
  30. +38 −0 vendor/github.com/evanphx/json-patch/errors.go
  31. +75 −44 vendor/github.com/evanphx/json-patch/patch.go

Some generated files are not rendered by default. Learn more.

Oops, something went wrong.
@@ -129,6 +129,8 @@ func TestAddFlags(t *testing.T) {
MaxMutatingRequestsInFlight: 200,
RequestTimeout: time.Duration(2) * time.Minute,
MinRequestTimeout: 1800,
JSONPatchMaxCopyBytes: int64(100 * 1024 * 1024),
MaxRequestBodyBytes: int64(100 * 1024 * 1024),
},
Admission: &kubeoptions.AdmissionOptions{
GenericAdmission: &apiserveroptions.AdmissionOptions{

Some generated files are not rendered by default. Learn more.

Oops, something went wrong.

Some generated files are not rendered by default. Learn more.

Oops, something went wrong.
@@ -327,6 +327,17 @@ func NewTooManyRequestsError(message string) *StatusError {
}}
}

// NewRequestEntityTooLargeError returns an error indicating that the request
// entity was too large.
func NewRequestEntityTooLargeError(message string) *StatusError {
return &StatusError{metav1.Status{
Status: metav1.StatusFailure,
Code: http.StatusRequestEntityTooLarge,
Reason: metav1.StatusReasonRequestEntityTooLarge,
Message: fmt.Sprintf("Request entity too large: %s", message),
}}
}

// NewGenericServerResponse returns a new error for server responses that are not in a recognizable form.
func NewGenericServerResponse(code int, verb string, qualifiedResource schema.GroupResource, name, serverMessage string, retryAfterSeconds int, isUnexpectedResponse bool) *StatusError {
reason := metav1.StatusReasonUnknown
@@ -513,6 +524,19 @@ func IsTooManyRequests(err error) bool {
return false
}

// IsRequestEntityTooLargeError determines if err is an error which indicates
// the request entity is too large.
func IsRequestEntityTooLargeError(err error) bool {
if ReasonForError(err) == metav1.StatusReasonRequestEntityTooLarge {
return true
}
switch t := err.(type) {
case APIStatus:
return t.Status().Code == http.StatusRequestEntityTooLarge
}
return false
}

// IsUnexpectedServerError returns true if the server response was not in the expected API format,
// and may be the result of another HTTP actor.
func IsUnexpectedServerError(err error) bool {
@@ -658,6 +658,10 @@ const (
// Status code 406
StatusReasonNotAcceptable StatusReason = "NotAcceptable"

// StatusReasonRequestEntityTooLarge means that the request entity is too large.
// Status code 413
StatusReasonRequestEntityTooLarge StatusReason = "RequestEntityTooLarge"

// StatusReasonUnsupportedMediaType means that the content type sent by the client is not acceptable
// to the server - for instance, attempting to send protobuf for a resource that supports only json and yaml.
// API calls that return UnsupportedMediaType can never succeed.

Some generated files are not rendered by default. Learn more.

Oops, something went wrong.
@@ -81,6 +81,10 @@ type APIGroupVersion struct {

// OpenAPIConfig lets the individual handlers build a subset of the OpenAPI schema before they are installed.
OpenAPIConfig *openapicommon.Config

// The limit on the request body size that would be accepted and decoded in a write request.
// 0 means no limit.
MaxRequestBodyBytes int64
}

// InstallREST registers the REST handlers (storage, watch, proxy and redirect) into a restful Container.
@@ -74,7 +74,7 @@ func createHandler(r rest.NamedCreater, scope RequestScope, admit admission.Inte
}
decoder := scope.Serializer.DecoderToVersion(s.Serializer, schema.GroupVersion{Group: gv.Group, Version: runtime.APIVersionInternal})

body, err := readBody(req)
body, err := limitedReadBody(req, scope.MaxRequestBodyBytes)
if err != nil {
scope.err(err, w, req)
return
@@ -61,7 +61,7 @@ func DeleteResource(r rest.GracefulDeleter, allowsOptions bool, scope RequestSco

options := &metav1.DeleteOptions{}
if allowsOptions {
body, err := readBody(req)
body, err := limitedReadBody(req, scope.MaxRequestBodyBytes)
if err != nil {
scope.err(err, w, req)
return
@@ -236,7 +236,7 @@ func DeleteCollection(r rest.CollectionDeleter, checkBody bool, scope RequestSco

options := &metav1.DeleteOptions{}
if checkBody {
body, err := readBody(req)
body, err := limitedReadBody(req, scope.MaxRequestBodyBytes)
if err != nil {
scope.err(err, w, req)
return
@@ -41,6 +41,11 @@ import (
utiltrace "k8s.io/apiserver/pkg/util/trace"
)

const (
// maximum number of operations a single json patch may contain.
maxJSONPatchOperations = 10000
)

// PatchResource returns a function that will handle a resource patch.
func PatchResource(r rest.Patcher, scope RequestScope, admit admission.Interface, patchTypes []string) http.HandlerFunc {
return func(w http.ResponseWriter, req *http.Request) {
@@ -82,7 +87,7 @@ func PatchResource(r rest.Patcher, scope RequestScope, admit admission.Interface
ctx := req.Context()
ctx = request.WithNamespace(ctx, namespace)

patchJS, err := readBody(req)
patchJS, err := limitedReadBody(req, scope.MaxRequestBodyBytes)
if err != nil {
scope.err(err, w, req)
return
@@ -241,9 +246,18 @@ func (p *jsonPatcher) applyJSPatch(versionedJS []byte) (patchedJS []byte, retErr
case types.JSONPatchType:
patchObj, err := jsonpatch.DecodePatch(p.patchJS)
if err != nil {
return nil, err
return nil, errors.NewBadRequest(err.Error())
}
if len(patchObj) > maxJSONPatchOperations {
return nil, errors.NewRequestEntityTooLargeError(
fmt.Sprintf("The allowed maximum operations in a JSON patch is %d, got %d",
maxJSONPatchOperations, len(patchObj)))
}
patchedJS, err := patchObj.Apply(versionedJS)
if err != nil {
return nil, errors.NewGenericServerResponse(http.StatusUnprocessableEntity, "", schema.GroupResource{}, "", err.Error(), 0, false)
}
return patchObj.Apply(versionedJS)
return patchedJS, nil
case types.MergePatchType:
return jsonpatch.MergePatch(versionedJS, p.patchJS)
default:
@@ -20,6 +20,7 @@ import (
"context"
"encoding/hex"
"fmt"
"io"
"io/ioutil"
"net/http"
"net/url"
@@ -64,6 +65,8 @@ type RequestScope struct {
Subresource string

MetaGroupVersion schema.GroupVersion

MaxRequestBodyBytes int64
}

func (scope *RequestScope) err(err error, w http.ResponseWriter, req *http.Request) {
@@ -317,9 +320,23 @@ func summarizeData(data []byte, maxLength int) string {
}
}

func readBody(req *http.Request) ([]byte, error) {
func limitedReadBody(req *http.Request, limit int64) ([]byte, error) {
defer req.Body.Close()
return ioutil.ReadAll(req.Body)
if limit <= 0 {
return ioutil.ReadAll(req.Body)
}
lr := &io.LimitedReader{
R: req.Body,
N: limit + 1,
}
data, err := ioutil.ReadAll(lr)
if err != nil {
return nil, err
}
if lr.N <= 0 {
return nil, errors.NewRequestEntityTooLargeError(fmt.Sprintf("limit is %d", limit))
}
return data, nil
}

func parseTimeout(str string) time.Duration {
@@ -56,7 +56,7 @@ func UpdateResource(r rest.Updater, scope RequestScope, admit admission.Interfac
ctx := req.Context()
ctx = request.WithNamespace(ctx, namespace)

body, err := readBody(req)
body, err := limitedReadBody(req, scope.MaxRequestBodyBytes)
if err != nil {
scope.err(err, w, req)
return
@@ -493,6 +493,8 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
Kind: fqKindToRegister,

MetaGroupVersion: metav1.SchemeGroupVersion,

MaxRequestBodyBytes: a.group.MaxRequestBodyBytes,
}
if a.group.MetaGroupVersion != nil {
reqScope.MetaGroupVersion = *a.group.MetaGroupVersion
@@ -93,6 +93,7 @@ go_library(
"//vendor/github.com/coreos/go-systemd/daemon:go_default_library",
"//vendor/github.com/emicklei/go-restful:go_default_library",
"//vendor/github.com/emicklei/go-restful-swagger12:go_default_library",
"//vendor/github.com/evanphx/json-patch:go_default_library",
"//vendor/github.com/go-openapi/spec:go_default_library",
"//vendor/github.com/golang/glog:go_default_library",
"//vendor/github.com/pborman/uuid:go_default_library",
@@ -27,9 +27,11 @@ import (
"sort"
"strconv"
"strings"
"sync/atomic"
"time"

"github.com/emicklei/go-restful-swagger12"
jsonpatch "github.com/evanphx/json-patch"
"github.com/go-openapi/spec"
"github.com/pborman/uuid"

@@ -159,6 +161,13 @@ type Config struct {
// If specified, long running requests such as watch will be allocated a random timeout between this value, and
// twice this value. Note that it is up to the request handlers to ignore or honor this timeout. In seconds.
MinRequestTimeout int
// The limit on the total size increase all "copy" operations in a json
// patch may cause.
// This affects all places that applies json patch in the binary.
JSONPatchMaxCopyBytes int64
// The limit on the request body size that would be accepted and decoded in a write request.
// 0 means no limit.
MaxRequestBodyBytes int64
// MaxRequestsInFlight is the maximum number of parallel non-long-running requests. Every further
// request has to wait. Applies only to non-mutating requests.
MaxRequestsInFlight int
@@ -249,21 +258,37 @@ type AuthorizationInfo struct {
// NewConfig returns a Config struct with the default values
func NewConfig(codecs serializer.CodecFactory) *Config {
return &Config{
Serializer: codecs,
ReadWritePort: 443,
BuildHandlerChainFunc: DefaultBuildHandlerChain,
HandlerChainWaitGroup: new(utilwaitgroup.SafeWaitGroup),
LegacyAPIGroupPrefixes: sets.NewString(DefaultLegacyAPIPrefix),
DisabledPostStartHooks: sets.NewString(),
HealthzChecks: []healthz.HealthzChecker{healthz.PingHealthz},
EnableIndex: true,
EnableDiscovery: true,
EnableProfiling: true,
EnableMetrics: true,
MaxRequestsInFlight: 400,
MaxMutatingRequestsInFlight: 200,
RequestTimeout: time.Duration(60) * time.Second,
MinRequestTimeout: 1800,
Serializer: codecs,
ReadWritePort: 443,
BuildHandlerChainFunc: DefaultBuildHandlerChain,
HandlerChainWaitGroup: new(utilwaitgroup.SafeWaitGroup),
LegacyAPIGroupPrefixes: sets.NewString(DefaultLegacyAPIPrefix),
DisabledPostStartHooks: sets.NewString(),
HealthzChecks: []healthz.HealthzChecker{healthz.PingHealthz},
EnableIndex: true,
EnableDiscovery: true,
EnableProfiling: true,
EnableMetrics: true,
MaxRequestsInFlight: 400,
MaxMutatingRequestsInFlight: 200,
RequestTimeout: time.Duration(60) * time.Second,
MinRequestTimeout: 1800,
// 10MB is the recommended maximum client request size in bytes
// the etcd server should accept. See
// https://github.com/etcd-io/etcd/blob/release-3.3/etcdserver/server.go#L90.
// A request body might be encoded in json, and is converted to
// proto when persisted in etcd. Assuming the upper bound of
// the size ratio is 10:1, we set 100MB as the largest size
// increase the "copy" operations in a json patch may cause.
JSONPatchMaxCopyBytes: int64(100 * 1024 * 1024),
// 10MB is the recommended maximum client request size in bytes
// the etcd server should accept. See
// https://github.com/etcd-io/etcd/blob/release-3.3/etcdserver/server.go#L90.
// A request body might be encoded in json, and is converted to
// proto when persisted in etcd. Assuming the upper bound of
// the size ratio is 10:1, we set 100MB as the largest request
// body size to be accepted and decoded in a write request.
MaxRequestBodyBytes: int64(100 * 1024 * 1024),
EnableAPIResponseCompression: utilfeature.DefaultFeatureGate.Enabled(features.APIResponseCompression),

// Default to treating watch as a long-running operation
@@ -489,6 +514,20 @@ func (c completedConfig) New(name string, delegationTarget DelegationTarget) (*G
DiscoveryGroupManager: discovery.NewRootAPIsHandler(c.DiscoveryAddresses, c.Serializer),

enableAPIResponseCompression: c.EnableAPIResponseCompression,
maxRequestBodyBytes: c.MaxRequestBodyBytes,
}

for {
if c.JSONPatchMaxCopyBytes <= 0 {
break
}
existing := atomic.LoadInt64(&jsonpatch.AccumulatedCopySizeLimit)
if existing > 0 && existing < c.JSONPatchMaxCopyBytes {
break
}
if atomic.CompareAndSwapInt64(&jsonpatch.AccumulatedCopySizeLimit, existing, c.JSONPatchMaxCopyBytes) {
break
}
}

for k, v := range delegationTarget.PostStartHooks() {
@@ -147,6 +147,10 @@ type GenericAPIServer struct {

// HandlerChainWaitGroup allows you to wait for all chain handlers finish after the server shutdown.
HandlerChainWaitGroup *utilwaitgroup.SafeWaitGroup

// The limit on the request body size that would be accepted and decoded in a write request.
// 0 means no limit.
maxRequestBodyBytes int64
}

// DelegationTarget is an interface which allows for composition of API servers with top level handling that works
@@ -324,6 +328,7 @@ func (s *GenericAPIServer) installAPIResources(apiPrefix string, apiGroupInfo *A
if apiGroupInfo.OptionsExternalVersion != nil {
apiGroupVersion.OptionsExternalVersion = apiGroupInfo.OptionsExternalVersion
}
apiGroupVersion.MaxRequestBodyBytes = s.maxRequestBodyBytes

if err := apiGroupVersion.InstallREST(s.Handler.GoRestfulContainer); err != nil {
return fmt.Errorf("unable to setup API %v: %v", apiGroupInfo, err)
Oops, something went wrong.

0 comments on commit 5d86c1b

Please sign in to comment.
You can’t perform that action at this time.