diff --git a/.gitignore b/.gitignore index 32efe9235..3f13f30a2 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,5 @@ bin *.swo *~ .output/ + +.staging diff --git a/constraint/Makefile b/constraint/Makefile index bff28a719..003897b42 100644 --- a/constraint/Makefile +++ b/constraint/Makefile @@ -42,6 +42,10 @@ manifests: paths="./pkg/..." \ output:crd:artifacts:config=config/crds kustomize build config/crds --output=deploy/crds.yaml + mkdir -p .staging/templatecrd + cp config/crds/* .staging/templatecrd + sed -i '/- externaldata.gatekeeper.sh_providers.yaml/d' .staging/templatecrd/kustomization.yaml + kustomize build .staging/templatecrd --output=.staging/templatecrd/crd.yaml lint: golangci-lint -v run ./... --timeout 5m @@ -76,10 +80,11 @@ YAML_CONSTANT_GOLANG_FILE := ./pkg/schema/yaml_constant.go constraint-template-string-constant: manifests rm -rf $(YAML_CONSTANT_GOLANG_FILE) bash -c 'echo -en ${FILE_STUB} >> ${YAML_CONSTANT_GOLANG_FILE}' - bash -c 'cat deploy/crds.yaml >> ${YAML_CONSTANT_GOLANG_FILE}' + bash -c 'cat .staging/templatecrd/crd.yaml >> ${YAML_CONSTANT_GOLANG_FILE}' bash -c 'echo "\`" >> ${YAML_CONSTANT_GOLANG_FILE}' # Remove trailing spaces. Double $ is to prevent variable expansion in make sed -i "s/ $$//g" ${YAML_CONSTANT_GOLANG_FILE} + rm -rf .staging generate-defaults: constraint-template-string-constant defaulter-gen \ diff --git a/constraint/config/crds/externaldata.gatekeeper.sh_providers.yaml b/constraint/config/crds/externaldata.gatekeeper.sh_providers.yaml new file mode 100644 index 000000000..2f1e935e1 --- /dev/null +++ b/constraint/config/crds/externaldata.gatekeeper.sh_providers.yaml @@ -0,0 +1,50 @@ + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.5.0 + creationTimestamp: null + name: providers.externaldata.gatekeeper.sh +spec: + group: externaldata.gatekeeper.sh + names: + kind: Provider + listKind: ProviderList + plural: providers + singular: provider + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: Provider is the Schema for the Provider API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: Spec defines the Provider specifications. + properties: + timeout: + description: Timeout is the timeout when querying the provider. + type: integer + url: + description: URL is the url for the provider. URL is prefixed with http:// or https://. + type: string + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/constraint/config/crds/kustomization.yaml b/constraint/config/crds/kustomization.yaml index b305727a9..43ff16d1b 100644 --- a/constraint/config/crds/kustomization.yaml +++ b/constraint/config/crds/kustomization.yaml @@ -1,5 +1,6 @@ resources: - templates.gatekeeper.sh_constrainttemplates.yaml +- externaldata.gatekeeper.sh_providers.yaml patchesStrategicMerge: - |- diff --git a/constraint/config/crds/templates.gatekeeper.sh_constrainttemplates.yaml b/constraint/config/crds/templates.gatekeeper.sh_constrainttemplates.yaml index 9325db1df..bb8bd8779 100644 --- a/constraint/config/crds/templates.gatekeeper.sh_constrainttemplates.yaml +++ b/constraint/config/crds/templates.gatekeeper.sh_constrainttemplates.yaml @@ -30,7 +30,7 @@ spec: metadata: type: object spec: - description: ConstraintTemplateSpec defines the desired state of ConstraintTemplate + description: ConstraintTemplateSpec defines the desired state of ConstraintTemplate. properties: crd: properties: @@ -73,7 +73,7 @@ spec: type: array type: object status: - description: ConstraintTemplateStatus defines the observed state of ConstraintTemplate + description: ConstraintTemplateStatus defines the observed state of ConstraintTemplate. properties: byPod: items: @@ -125,7 +125,7 @@ spec: metadata: type: object spec: - description: ConstraintTemplateSpec defines the desired state of ConstraintTemplate + description: ConstraintTemplateSpec defines the desired state of ConstraintTemplate. properties: crd: properties: @@ -168,7 +168,7 @@ spec: type: array type: object status: - description: ConstraintTemplateStatus defines the observed state of ConstraintTemplate + description: ConstraintTemplateStatus defines the observed state of ConstraintTemplate. properties: byPod: items: @@ -220,7 +220,7 @@ spec: metadata: type: object spec: - description: ConstraintTemplateSpec defines the desired state of ConstraintTemplate + description: ConstraintTemplateSpec defines the desired state of ConstraintTemplate. properties: crd: properties: @@ -263,7 +263,7 @@ spec: type: array type: object status: - description: ConstraintTemplateStatus defines the observed state of ConstraintTemplate + description: ConstraintTemplateStatus defines the observed state of ConstraintTemplate. properties: byPod: items: diff --git a/constraint/config/kustomization.yaml b/constraint/config/kustomization.yaml new file mode 100644 index 000000000..50d828dfe --- /dev/null +++ b/constraint/config/kustomization.yaml @@ -0,0 +1,3 @@ +resources: +- crds/templates.gatekeeper.sh_constrainttemplates.yaml +- crds/externaldata.gatekeeper.sh_providers.yaml diff --git a/constraint/deploy/crds.yaml b/constraint/deploy/crds.yaml index 61a1d9d35..b1fd342c4 100644 --- a/constraint/deploy/crds.yaml +++ b/constraint/deploy/crds.yaml @@ -28,7 +28,7 @@ spec: metadata: type: object spec: - description: ConstraintTemplateSpec defines the desired state of ConstraintTemplate + description: ConstraintTemplateSpec defines the desired state of ConstraintTemplate. properties: crd: properties: @@ -71,7 +71,7 @@ spec: type: array type: object status: - description: ConstraintTemplateStatus defines the observed state of ConstraintTemplate + description: ConstraintTemplateStatus defines the observed state of ConstraintTemplate. properties: byPod: items: @@ -123,7 +123,7 @@ spec: metadata: type: object spec: - description: ConstraintTemplateSpec defines the desired state of ConstraintTemplate + description: ConstraintTemplateSpec defines the desired state of ConstraintTemplate. properties: crd: properties: @@ -166,7 +166,7 @@ spec: type: array type: object status: - description: ConstraintTemplateStatus defines the observed state of ConstraintTemplate + description: ConstraintTemplateStatus defines the observed state of ConstraintTemplate. properties: byPod: items: @@ -218,7 +218,7 @@ spec: metadata: type: object spec: - description: ConstraintTemplateSpec defines the desired state of ConstraintTemplate + description: ConstraintTemplateSpec defines the desired state of ConstraintTemplate. properties: crd: properties: @@ -261,7 +261,7 @@ spec: type: array type: object status: - description: ConstraintTemplateStatus defines the observed state of ConstraintTemplate + description: ConstraintTemplateStatus defines the observed state of ConstraintTemplate. properties: byPod: items: @@ -305,3 +305,52 @@ status: plural: "" conditions: [] storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.5.0 + creationTimestamp: null + name: providers.externaldata.gatekeeper.sh +spec: + group: externaldata.gatekeeper.sh + names: + kind: Provider + listKind: ProviderList + plural: providers + singular: provider + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: Provider is the Schema for the Provider API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: Spec defines the Provider specifications. + properties: + timeout: + description: Timeout is the timeout when querying the provider. + type: integer + url: + description: URL is the url for the provider. URL is prefixed with http:// or https://. + type: string + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/constraint/pkg/apis/addtoscheme_externaldata_v1alpha1.go b/constraint/pkg/apis/addtoscheme_externaldata_v1alpha1.go new file mode 100644 index 000000000..88def6752 --- /dev/null +++ b/constraint/pkg/apis/addtoscheme_externaldata_v1alpha1.go @@ -0,0 +1,25 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package apis + +import ( + "github.com/open-policy-agent/frameworks/constraint/pkg/apis/externaldata/v1alpha1" +) + +func init() { + // Register the types with the Scheme so the components can map objects to GroupVersionKinds and back + AddToSchemes = append(AddToSchemes, v1alpha1.AddToScheme) +} diff --git a/constraint/pkg/apis/externaldata/v1alpha1/doc.go b/constraint/pkg/apis/externaldata/v1alpha1/doc.go new file mode 100644 index 000000000..c1f0af513 --- /dev/null +++ b/constraint/pkg/apis/externaldata/v1alpha1/doc.go @@ -0,0 +1,21 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package v1alpha1 contains API Schema definitions for the externaldata v1alpha1 API group +// +k8s:openapi-gen=true +// +k8s:deepcopy-gen=package,register +// +k8s:defaulter-gen=TypeMeta +// +groupName=externaldata.gatekeeper.sh +package v1alpha1 diff --git a/constraint/pkg/apis/externaldata/v1alpha1/provider_types.go b/constraint/pkg/apis/externaldata/v1alpha1/provider_types.go new file mode 100644 index 000000000..5e1395192 --- /dev/null +++ b/constraint/pkg/apis/externaldata/v1alpha1/provider_types.go @@ -0,0 +1,58 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// ProviderSpec defines the desired state of Provider. +type ProviderSpec struct { + // URL is the url for the provider. URL is prefixed with http:// or https://. + URL string `json:"url,omitempty"` + // Timeout is the timeout when querying the provider. + Timeout int `json:"timeout,omitempty"` +} + +// +genclient +// +genclient:nonNamespaced +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:resource:scope=Cluster + +// Provider is the Schema for the Provider API +// +k8s:openapi-gen=true +type Provider struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + // Spec defines the Provider specifications. + Spec ProviderSpec `json:"spec,omitempty"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// ProviderList contains a list of Provider. +type ProviderList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + + // Items contains the list of Providers. + Items []Provider `json:"items"` +} + +func init() { + SchemeBuilder.Register(&Provider{}, &ProviderList{}) +} diff --git a/constraint/pkg/apis/externaldata/v1alpha1/provider_types_test.go b/constraint/pkg/apis/externaldata/v1alpha1/provider_types_test.go new file mode 100644 index 000000000..1036ce46d --- /dev/null +++ b/constraint/pkg/apis/externaldata/v1alpha1/provider_types_test.go @@ -0,0 +1,16 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 diff --git a/constraint/pkg/apis/externaldata/v1alpha1/register.go b/constraint/pkg/apis/externaldata/v1alpha1/register.go new file mode 100644 index 000000000..793ba1cf4 --- /dev/null +++ b/constraint/pkg/apis/externaldata/v1alpha1/register.go @@ -0,0 +1,47 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// NOTE: Boilerplate only. Ignore this file. + +// Package v1alpha1 contains API Schema definitions for the externaldata v1alpha1 API group +// +k8s:openapi-gen=true +// +k8s:deepcopy-gen=package,register +// +k8s:conversion-gen=github.com/open-policy-agent/frameworks/constraint/pkg/apis/externaldata +// +k8s:defaulter-gen=TypeMeta +// +groupName=externaldata.gatekeeper.sh +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +var ( + // SchemeGroupVersion is group version used to register these objects. + SchemeGroupVersion = schema.GroupVersion{Group: "externaldata.gatekeeper.sh", Version: "v1alpha1"} + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme. + SchemeBuilder = &scheme.Builder{GroupVersion: SchemeGroupVersion} + + localSchemeBuilder = runtime.NewSchemeBuilder(SchemeBuilder.AddToScheme) + + AddToScheme = localSchemeBuilder.AddToScheme +) + +// Resource is required by pkg/client/listers/... +func Resource(resource string) schema.GroupResource { + return SchemeGroupVersion.WithResource(resource).GroupResource() +} diff --git a/constraint/pkg/apis/externaldata/v1alpha1/v1alpha1_suite_test.go b/constraint/pkg/apis/externaldata/v1alpha1/v1alpha1_suite_test.go new file mode 100644 index 000000000..972922e1b --- /dev/null +++ b/constraint/pkg/apis/externaldata/v1alpha1/v1alpha1_suite_test.go @@ -0,0 +1,58 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "log" + "os" + "path/filepath" + "testing" + + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" +) + +var ( + cfg *rest.Config + c client.Client +) + +func TestMain(m *testing.M) { + t := &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "..", "deploy", "crds.yaml")}, + } + + err := SchemeBuilder.AddToScheme(scheme.Scheme) + if err != nil { + log.Fatal(err) + } + + if cfg, err = t.Start(); err != nil { + log.Fatal(err) + } + + if c, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}); err != nil { + log.Fatal(err) + } + + code := m.Run() + if err := t.Stop(); err != nil { + log.Fatal(err) + } + os.Exit(code) +} diff --git a/constraint/pkg/apis/externaldata/v1alpha1/zz_generated.deepcopy.go b/constraint/pkg/apis/externaldata/v1alpha1/zz_generated.deepcopy.go new file mode 100644 index 000000000..740e304d1 --- /dev/null +++ b/constraint/pkg/apis/externaldata/v1alpha1/zz_generated.deepcopy.go @@ -0,0 +1,97 @@ +// +build !ignore_autogenerated + +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by controller-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Provider) DeepCopyInto(out *Provider) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Provider. +func (in *Provider) DeepCopy() *Provider { + if in == nil { + return nil + } + out := new(Provider) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *Provider) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ProviderList) DeepCopyInto(out *ProviderList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]Provider, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProviderList. +func (in *ProviderList) DeepCopy() *ProviderList { + if in == nil { + return nil + } + out := new(ProviderList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ProviderList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ProviderSpec) DeepCopyInto(out *ProviderSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProviderSpec. +func (in *ProviderSpec) DeepCopy() *ProviderSpec { + if in == nil { + return nil + } + out := new(ProviderSpec) + in.DeepCopyInto(out) + return out +} diff --git a/constraint/pkg/client/drivers/local/local.go b/constraint/pkg/client/drivers/local/local.go index 3c4e3351c..d0276d600 100644 --- a/constraint/pkg/client/drivers/local/local.go +++ b/constraint/pkg/client/drivers/local/local.go @@ -6,16 +6,21 @@ import ( "encoding/json" "errors" "fmt" + "io/ioutil" + "net/http" "strings" "sync" + "time" "github.com/open-policy-agent/frameworks/constraint/pkg/client/drivers" + "github.com/open-policy-agent/frameworks/constraint/pkg/externaldata" "github.com/open-policy-agent/frameworks/constraint/pkg/types" "github.com/open-policy-agent/opa/ast" "github.com/open-policy-agent/opa/rego" "github.com/open-policy-agent/opa/storage" "github.com/open-policy-agent/opa/storage/inmem" "github.com/open-policy-agent/opa/topdown" + opatypes "github.com/open-policy-agent/opa/types" ) const ( @@ -67,6 +72,12 @@ func DisableBuiltins(builtins ...string) Arg { } } +func AddExternalDataProviderCache(providerCache *externaldata.ProviderCache) Arg { + return func(d *driver) { + d.providerCache = providerCache + } +} + func New(args ...Arg) drivers.Driver { d := &driver{ compiler: ast.NewCompiler(), @@ -77,6 +88,15 @@ func New(args ...Arg) drivers.Driver { for _, arg := range args { arg(d) } + + // adding externaldata builtin otherwise capabilities get overridden + // if a capability, like http.send, is disabled + if d.providerCache != nil { + d.capabilities.Builtins = append(d.capabilities.Builtins, &ast.Builtin{ + Name: "external_data", + Decl: opatypes.NewFunction(opatypes.Args(opatypes.A), opatypes.A), + }) + } d.compiler.WithCapabilities(d.capabilities) return d } @@ -84,15 +104,82 @@ func New(args ...Arg) drivers.Driver { var _ drivers.Driver = &driver{} type driver struct { - modulesMux sync.RWMutex - compiler *ast.Compiler - modules map[string]*ast.Module - storage storage.Store - capabilities *ast.Capabilities - traceEnabled bool + modulesMux sync.RWMutex + compiler *ast.Compiler + modules map[string]*ast.Module + storage storage.Store + capabilities *ast.Capabilities + traceEnabled bool + providerCache *externaldata.ProviderCache } func (d *driver) Init(ctx context.Context) error { + if d.providerCache != nil { + rego.RegisterBuiltin1( + ®o.Function{ + Name: "external_data", + Decl: opatypes.NewFunction(opatypes.Args(opatypes.A), opatypes.A), + Memoize: true, + }, + func(bctx rego.BuiltinContext, regorequest *ast.Term) (*ast.Term, error) { + var regoReq externaldata.RegoRequest + if err := ast.As(regorequest.Value, ®oReq); err != nil { + return nil, err + } + // only primitive types are allowed for keys + for _, key := range regoReq.Keys { + switch v := key.(type) { + case int: + case int32: + case int64: + case string: + case float64: + case float32: + break + default: + return externaldata.HandleError(http.StatusBadRequest, fmt.Errorf("type %v is not supported in external_data", v)) + } + } + + provider, err := d.providerCache.Get(regoReq.ProviderName) + if err != nil { + return externaldata.HandleError(http.StatusBadRequest, err) + } + + externaldataRequest := externaldata.NewProviderRequest(regoReq.Keys) + reqBody, err := json.Marshal(externaldataRequest) + if err != nil { + return externaldata.HandleError(http.StatusInternalServerError, err) + } + + req, err := http.NewRequest("POST", provider.Spec.URL, bytes.NewBuffer(reqBody)) + if err != nil { + return externaldata.HandleError(http.StatusInternalServerError, err) + } + + ctx, cancel := context.WithDeadline(bctx.Context, time.Now().Add(time.Duration(provider.Spec.Timeout)*time.Second)) + defer cancel() + + resp, err := http.DefaultClient.Do(req.WithContext(ctx)) + if err != nil { + return externaldata.HandleError(http.StatusInternalServerError, err) + } + defer resp.Body.Close() + respBody, err := ioutil.ReadAll(resp.Body) + if err != nil { + return externaldata.HandleError(http.StatusInternalServerError, err) + } + + var externaldataResponse externaldata.ProviderResponse + if err := json.Unmarshal(respBody, &externaldataResponse); err != nil { + return externaldata.HandleError(http.StatusInternalServerError, err) + } + + regoResponse := externaldata.NewRegoResponse(resp.StatusCode, &externaldataResponse) + return externaldata.PrepareRegoResponse(regoResponse) + }, + ) + } return nil } diff --git a/constraint/pkg/client/drivers/local/local_test.go b/constraint/pkg/client/drivers/local/local_test.go index 5caf361ca..c0b353b47 100644 --- a/constraint/pkg/client/drivers/local/local_test.go +++ b/constraint/pkg/client/drivers/local/local_test.go @@ -11,6 +11,7 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/open-policy-agent/frameworks/constraint/pkg/client/drivers" + "github.com/open-policy-agent/frameworks/constraint/pkg/externaldata" "github.com/open-policy-agent/frameworks/constraint/pkg/types" "github.com/open-policy-agent/opa/rego" ) @@ -172,6 +173,7 @@ func resultsEqual(res rego.ResultSet, exp []string, t *testing.T) bool { } func TestModules(t *testing.T) { + providerCache := externaldata.NewCache() tc := []compositeTestCase{ { Name: "PutModules then DeleteModules", @@ -318,7 +320,7 @@ func TestModules(t *testing.T) { Op: putModules, RuleNamePrefix: "test1", Rules: rules{ - {Content: `package hello a = http.send({"method": "get", "url": "https://github.com/"}) `}, + {Content: `package hello a = http.send({"method": "get", "url": "https://github.com/"})`}, }, ErrorExpected: false, }, @@ -331,13 +333,41 @@ func TestModules(t *testing.T) { Op: putModules, RuleNamePrefix: "test1", Rules: rules{ - {Content: `package hello a = http.send({"method": "get", "url": "https://github.com/"}) `}, + {Content: `package hello a = http.send({"method": "get", "url": "https://github.com/"})`}, }, ErrorExpected: true, }, }, driverArg: []Arg{DisableBuiltins("http.send")}, }, + { + Name: "PutModule with external data cache", + Actions: []*action{ + { + Op: putModules, + RuleNamePrefix: "test1", + Rules: rules{ + {Content: `package hello a = external_data({"provider": "my-provider", "keys": ["foo", 123]})`}, + }, + ErrorExpected: false, + }, + }, + driverArg: []Arg{AddExternalDataProviderCache(providerCache)}, + }, + { + Name: "PutModule with external data disabled", + Actions: []*action{ + { + Op: putModules, + RuleNamePrefix: "test1", + Rules: rules{ + {Content: `package hello a = external_data({"provider": "my-provider", "keys": ["foo", 123]})`}, + }, + ErrorExpected: true, + }, + }, + driverArg: []Arg{DisableBuiltins("external_data")}, + }, } for _, tt := range tc { t.Run(tt.Name, tt.run) diff --git a/constraint/pkg/externaldata/cache.go b/constraint/pkg/externaldata/cache.go new file mode 100644 index 000000000..6062fb972 --- /dev/null +++ b/constraint/pkg/externaldata/cache.go @@ -0,0 +1,74 @@ +package externaldata + +import ( + "fmt" + "strings" + "sync" + + "github.com/open-policy-agent/frameworks/constraint/pkg/apis/externaldata/v1alpha1" +) + +type ProviderCache struct { + cache map[string]v1alpha1.Provider + mux sync.RWMutex +} + +func NewCache() *ProviderCache { + return &ProviderCache{ + cache: make(map[string]v1alpha1.Provider), + } +} + +func (c *ProviderCache) Get(key string) (v1alpha1.Provider, error) { + c.mux.RLock() + defer c.mux.RUnlock() + + if v, ok := c.cache[key]; ok { + dc := *v.DeepCopy() + return dc, nil + } + return v1alpha1.Provider{}, fmt.Errorf("key is not found in provider cache") +} + +func (c *ProviderCache) Upsert(provider *v1alpha1.Provider) error { + c.mux.Lock() + defer c.mux.Unlock() + + if !isValidName(provider.Name) { + return fmt.Errorf("provider name can not be empty. value %s", provider.Name) + } + if !isValidURL(provider.Spec.URL) { + return fmt.Errorf("invalid provider url. value: %s", provider.Spec.URL) + } + if !isValidTimeout(provider.Spec.Timeout) { + return fmt.Errorf("provider timeout should be a positive integer. value: %d", provider.Spec.Timeout) + } + + c.cache[provider.GetName()] = *provider.DeepCopy() + return nil +} + +func (c *ProviderCache) Remove(name string) { + c.mux.Lock() + defer c.mux.Unlock() + + delete(c.cache, name) +} + +func isValidName(name string) bool { + return len(name) != 0 +} + +func isValidURL(url string) bool { + if len(url) == 0 { + return false + } + if !strings.HasPrefix(url, "http://") && !strings.HasPrefix(url, "https://") { + return false + } + return true +} + +func isValidTimeout(timeout int) bool { + return timeout >= 0 +} diff --git a/constraint/pkg/externaldata/cache_test.go b/constraint/pkg/externaldata/cache_test.go new file mode 100644 index 000000000..6cebd7223 --- /dev/null +++ b/constraint/pkg/externaldata/cache_test.go @@ -0,0 +1,129 @@ +package externaldata + +import ( + "testing" + + "github.com/open-policy-agent/frameworks/constraint/pkg/apis/externaldata/v1alpha1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type cacheTestCase struct { + Name string + Provider *v1alpha1.Provider + ErrorExpected bool +} + +func createProvider(name string, url string, timeout int) *v1alpha1.Provider { + return &v1alpha1.Provider{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Spec: v1alpha1.ProviderSpec{ + URL: url, + Timeout: timeout, + }, + } +} + +func TestUpsert(t *testing.T) { + tc := []cacheTestCase{ + { + Name: "valid http provider", + Provider: createProvider("test", "http://test", 1), + ErrorExpected: false, + }, + { + Name: "valid https provider", + Provider: createProvider("test", "https://test", 1), + ErrorExpected: false, + }, + { + Name: "empty name", + Provider: createProvider("", "http://test", 1), + ErrorExpected: true, + }, + { + Name: "empty url", + Provider: createProvider("test", "", 1), + ErrorExpected: true, + }, + { + Name: "invalid url", + Provider: createProvider("test", "gopher://test", 1), + ErrorExpected: true, + }, + { + Name: "invalid timeout", + Provider: createProvider("test", "http://test", -1), + ErrorExpected: true, + }, + { + Name: "empty provider", + Provider: &v1alpha1.Provider{}, + ErrorExpected: true, + }, + } + for _, tt := range tc { + cache := NewCache() + t.Run(tt.Name, func(t *testing.T) { + err := cache.Upsert(tt.Provider) + + if (err == nil) && tt.ErrorExpected { + t.Fatalf("err = nil; want non-nil") + } + if (err != nil) && !tt.ErrorExpected { + t.Fatalf("err = \"%s\"; want nil", err) + } + }) + } +} + +func TestGet(t *testing.T) { + tc := []cacheTestCase{ + { + Name: "valid provider", + Provider: createProvider("test", "http://test", 1), + ErrorExpected: false, + }, + { + Name: "invalid provider", + Provider: createProvider("", "http://test", 1), + ErrorExpected: true, + }, + } + for _, tt := range tc { + cache := NewCache() + t.Run(tt.Name, func(t *testing.T) { + _ = cache.Upsert(tt.Provider) + _, err := cache.Get(tt.Provider.Name) + + if (err == nil) && tt.ErrorExpected { + t.Fatalf("err = nil; want non-nil") + } + if (err != nil) && !tt.ErrorExpected { + t.Fatalf("err = \"%s\"; want nil", err) + } + }) + } +} + +func TestRemove(t *testing.T) { + tc := []cacheTestCase{ + { + Name: "valid provider", + Provider: createProvider("test", "http://test", 1), + ErrorExpected: false, + }, + } + for _, tt := range tc { + cache := NewCache() + t.Run(tt.Name, func(t *testing.T) { + _ = cache.Upsert(tt.Provider) + cache.Remove(tt.Provider.Name) + + if (cache != nil) && tt.ErrorExpected { + t.Fatalf("cache = \"%v\"; want nil", cache) + } + }) + } +} diff --git a/constraint/pkg/externaldata/request.go b/constraint/pkg/externaldata/request.go new file mode 100644 index 000000000..b8337f4a3 --- /dev/null +++ b/constraint/pkg/externaldata/request.go @@ -0,0 +1,46 @@ +package externaldata + +// RegoRequest is the request for external_data rego function. +type RegoRequest struct { + // ProviderName is the name of the external data provider. + ProviderName string `json:"provider"` + // Keys is the list of keys to send to the external data provider. + Keys []interface{} `json:"keys"` +} + +// ProviderRequest is the API request for the external data provider. +type ProviderRequest struct { + // APIVersion is the API version of the external data provider. + APIVersion string `json:"apiVersion,omitempty"` + // Kind is kind of the external data provider API call. This can be "ProviderRequest" or "ProviderResponse". + Kind ProviderKind `json:"kind,omitempty"` + // Request contains the request for the external data provider. + Request Request `json:"request,omitempty"` +} + +// Request is the struct that contains the keys to query. +type Request struct { + // Keys is the list of keys to send to the external data provider. + Keys []interface{} `json:"keys,omitempty"` +} + +// NewRequest creates a new request for the external data provider. +func NewProviderRequest(keys []interface{}) *ProviderRequest { + return &ProviderRequest{ + APIVersion: "externaldata.gatekeeper.sh/v1alpha1", + Kind: "ProviderRequest", + Request: Request{ + Keys: keys, + }, + } +} + +// +kubebuilder:validation:Enum=ProviderRequestKind;ProviderResponseKind +type ProviderKind string + +const ( + // ProviderRequestKind is the kind of the request. + ProviderRequestKind ProviderKind = "ProviderRequest" + // ProviderResponseKind is the kind of the response. + ProviderResponseKind ProviderKind = "ProviderResponse" +) diff --git a/constraint/pkg/externaldata/response.go b/constraint/pkg/externaldata/response.go new file mode 100644 index 000000000..6c34c9c37 --- /dev/null +++ b/constraint/pkg/externaldata/response.go @@ -0,0 +1,94 @@ +package externaldata + +import ( + "bytes" + "encoding/json" + + "github.com/open-policy-agent/opa/ast" +) + +// RegoResponse is the response inside rego. +type RegoResponse struct { + // Responses contains the response from the provider. + // In each element of the outer array, the first element is the key and the second is the corresponding value from the provider. + Responses [][]interface{} `json:"responses"` + // Errors contains the errors from the provider. + // In each item of the outer array, the first element is the key and the second is the corresponding error from the provider. + Errors [][]interface{} `json:"errors"` + // StatusCode contains the status code of the response. + StatusCode int `json:"status_code"` + // SystemError is the system error of the response. + SystemError string `json:"system_error"` +} + +// ProviderResponse is the API response from a provider. +type ProviderResponse struct { + // APIVersion is the API version of the external data provider. + APIVersion string `json:"apiVersion,omitempty"` + // Kind is kind of the external data provider API call. This can be "ProviderRequest" or "ProviderResponse". + Kind ProviderKind `json:"kind,omitempty"` + // Response contains the response from the provider. + Response Response `json:"response,omitempty"` +} + +// Response is the struct that holds the response from a provider. +type Response struct { + // Idempotent indicates that the responses from the provider are idempotent. + // Applies to mutation only and must be true for mutation. + Idempotent bool `json:"idempotent,omitempty"` + // Items contains the key, value and error from the provider. + Items []Item `json:"items,omitempty"` + // SystemError is the system error of the response. + SystemError string `json:"systemError,omitempty"` +} + +// Items is the struct that contains the key, value or error from a provider response. +type Item struct { + // Key is the request from the provider. + Key interface{} `json:"key,omitempty"` + // Value is the response from the provider. + Value interface{} `json:"value,omitempty"` + // Error is the error from the provider. + Error string `json:"error,omitempty"` +} + +// NewRegoResponse creates a new rego response from the given provider response. +func NewRegoResponse(statusCode int, pr *ProviderResponse) *RegoResponse { + responses := make([][]interface{}, 0) + errors := make([][]interface{}, 0) + + for _, item := range pr.Response.Items { + if item.Error != "" { + errors = append(errors, []interface{}{item.Key, item.Error}) + } else { + responses = append(responses, []interface{}{item.Key, item.Value}) + } + } + + return &RegoResponse{ + Responses: responses, + Errors: errors, + StatusCode: statusCode, + SystemError: pr.Response.SystemError, + } +} + +func PrepareRegoResponse(regoResponse *RegoResponse) (*ast.Term, error) { + rr, err := json.Marshal(regoResponse) + if err != nil { + return nil, err + } + v, err := ast.ValueFromReader(bytes.NewReader(rr)) + if err != nil { + return nil, err + } + return ast.NewTerm(v), nil +} + +func HandleError(statusCode int, err error) (*ast.Term, error) { + regoResponse := RegoResponse{ + StatusCode: statusCode, + SystemError: err.Error(), + } + return PrepareRegoResponse(®oResponse) +} diff --git a/constraint/pkg/schema/yaml_constant.go b/constraint/pkg/schema/yaml_constant.go index 6b79d3059..7ef2dd88f 100644 --- a/constraint/pkg/schema/yaml_constant.go +++ b/constraint/pkg/schema/yaml_constant.go @@ -33,7 +33,7 @@ spec: metadata: type: object spec: - description: ConstraintTemplateSpec defines the desired state of ConstraintTemplate + description: ConstraintTemplateSpec defines the desired state of ConstraintTemplate. properties: crd: properties: @@ -76,7 +76,7 @@ spec: type: array type: object status: - description: ConstraintTemplateStatus defines the observed state of ConstraintTemplate + description: ConstraintTemplateStatus defines the observed state of ConstraintTemplate. properties: byPod: items: @@ -128,7 +128,7 @@ spec: metadata: type: object spec: - description: ConstraintTemplateSpec defines the desired state of ConstraintTemplate + description: ConstraintTemplateSpec defines the desired state of ConstraintTemplate. properties: crd: properties: @@ -171,7 +171,7 @@ spec: type: array type: object status: - description: ConstraintTemplateStatus defines the observed state of ConstraintTemplate + description: ConstraintTemplateStatus defines the observed state of ConstraintTemplate. properties: byPod: items: @@ -223,7 +223,7 @@ spec: metadata: type: object spec: - description: ConstraintTemplateSpec defines the desired state of ConstraintTemplate + description: ConstraintTemplateSpec defines the desired state of ConstraintTemplate. properties: crd: properties: @@ -266,7 +266,7 @@ spec: type: array type: object status: - description: ConstraintTemplateStatus defines the observed state of ConstraintTemplate + description: ConstraintTemplateStatus defines the observed state of ConstraintTemplate. properties: byPod: items: