diff --git a/examples/ignore/README.md b/examples/ignore/README.md new file mode 100644 index 00000000..2bf653cf --- /dev/null +++ b/examples/ignore/README.md @@ -0,0 +1 @@ +`vendir sync` will populate `vendor` directory directly. diff --git a/examples/ignore/vendir.lock.yml b/examples/ignore/vendir.lock.yml new file mode 100755 index 00000000..9d70bb28 --- /dev/null +++ b/examples/ignore/vendir.lock.yml @@ -0,0 +1,9 @@ +apiVersion: vendir.k14s.io/v1alpha1 +directories: +- contents: + - git: + commitTitle: 'feat: add /metrics prometheus scrapable endpoint...' + sha: 2b009b61fa8afb330a4302c694ee61b11104c54c + path: . + path: vendor +kind: LockConfig diff --git a/examples/ignore/vendir.yml b/examples/ignore/vendir.yml new file mode 100644 index 00000000..d32c0da5 --- /dev/null +++ b/examples/ignore/vendir.yml @@ -0,0 +1,14 @@ +apiVersion: vendir.k14s.io/v1alpha1 +kind: Config +directories: +- path: vendor + contents: + - path: . + git: + url: https://github.com/cloudfoundry/cf-k8s-networking + ref: 2b009b61fa8afb330a4302c694ee61b11104c54c + includePaths: + - cfroutesync/crds/**/* + - install/ytt/networking/**/* + ignorePaths: + - install/ytt/**/values.yaml diff --git a/examples/ignore/vendor/LICENSE b/examples/ignore/vendor/LICENSE new file mode 100644 index 00000000..261eeb9e --- /dev/null +++ b/examples/ignore/vendor/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/examples/ignore/vendor/NOTICE b/examples/ignore/vendor/NOTICE new file mode 100644 index 00000000..a5a155ab --- /dev/null +++ b/examples/ignore/vendor/NOTICE @@ -0,0 +1,8 @@ +Copyright (c) 2019-Present CloudFoundry.org Foundation, Inc. All Rights Reserved. + +This project is licensed to you under the Apache License, Version 2.0 (the "License"). +You may not use this project except in compliance with the License. + +This project may include a number of subcomponents with separate copyright notices +and license terms. Your use of these subcomponents is subject to the terms and +conditions of the subcomponent's license, as noted in the LICENSE file. diff --git a/examples/ignore/vendor/cfroutesync/crds/routebulksync.yaml b/examples/ignore/vendor/cfroutesync/crds/routebulksync.yaml new file mode 100644 index 00000000..85aa2ca3 --- /dev/null +++ b/examples/ignore/vendor/cfroutesync/crds/routebulksync.yaml @@ -0,0 +1,28 @@ +apiVersion: "apiextensions.k8s.io/v1beta1" +kind: "CustomResourceDefinition" +metadata: + name: routebulksyncs.apps.cloudfoundry.org +spec: + group: apps.cloudfoundry.org + versions: + - name: v1alpha1 + served: true + storage: true + scope: Namespaced + names: + plural: routebulksyncs + singular: routebulksync + kind: RouteBulkSync + subresources: + status: {} + validation: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + selector: + type: object + template: + type: object diff --git a/examples/ignore/vendor/install/ytt/networking/templates/cfroutesync-configmap.yaml b/examples/ignore/vendor/install/ytt/networking/templates/cfroutesync-configmap.yaml new file mode 100644 index 00000000..14a4cda1 --- /dev/null +++ b/examples/ignore/vendor/install/ytt/networking/templates/cfroutesync-configmap.yaml @@ -0,0 +1,14 @@ +#@ load("@ytt:data", "data") +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: cfroutesync-config + namespace: #@ data.values.systemNamespace +data: + ccBaseURL: #@ data.values.cfroutesync.ccBaseURL + uaaBaseURL: #@ data.values.cfroutesync.uaaBaseURL + ccCA: #@ data.values.cfroutesync.ccCA + uaaCA: #@ data.values.cfroutesync.uaaCA + clientName: #@ data.values.cfroutesync.clientName + eiriniPodLabelPrefix: #@ data.values.cfroutesync.eiriniPodLabelPrefix diff --git a/examples/ignore/vendor/install/ytt/networking/templates/cfroutesync-secret.yaml b/examples/ignore/vendor/install/ytt/networking/templates/cfroutesync-secret.yaml new file mode 100644 index 00000000..f77fecf7 --- /dev/null +++ b/examples/ignore/vendor/install/ytt/networking/templates/cfroutesync-secret.yaml @@ -0,0 +1,10 @@ +#@ load("@ytt:data", "data") +--- +apiVersion: v1 +kind: Secret +metadata: + name: cfroutesync + namespace: #@ data.values.systemNamespace +type: Opaque +stringData: + clientSecret: #@ data.values.cfroutesync.clientSecret diff --git a/examples/ignore/vendor/install/ytt/networking/templates/cfroutesync.yaml b/examples/ignore/vendor/install/ytt/networking/templates/cfroutesync.yaml new file mode 100644 index 00000000..7f040f24 --- /dev/null +++ b/examples/ignore/vendor/install/ytt/networking/templates/cfroutesync.yaml @@ -0,0 +1,86 @@ +#@ load("@ytt:data", "data") + +#@ def labels(): +app: cfroutesync +#@ end + +--- +apiVersion: metacontroller.k8s.io/v1alpha1 +kind: CompositeController +metadata: + name: cfroutesync +spec: + resyncPeriodSeconds: 5 + parentResource: + apiVersion: apps.cloudfoundry.org/v1alpha1 + resource: routebulksyncs + childResources: + - apiVersion: v1 + resource: services + updateStrategy: + method: InPlace + - apiVersion: networking.istio.io/v1alpha3 + resource: virtualservices + updateStrategy: + method: InPlace + hooks: + sync: + webhook: + url: #@ "http://cfroutesync.{}/sync".format(data.values.systemNamespace) +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cfroutesync + namespace: #@ data.values.systemNamespace + annotations: + kapp.k14s.io/update-strategy: fallback-on-replace +spec: + selector: + matchLabels: #@ labels() + replicas: 1 + template: + metadata: + labels: #@ labels() + spec: + containers: + - name: cfroutesync + image: #@ data.values.cfroutesync.image + args: [ "-c", "/etc/cfroutesync-config"] + imagePullPolicy: Always + envFrom: + - configMapRef: + name: cfroutesync-config + volumeMounts: + - name: cfroutesync-credentials + mountPath: /etc/cfroutesync-config + readOnly: true + volumes: + - name: cfroutesync-credentials + secret: + secretName: cfroutesync +--- +apiVersion: v1 +kind: Service +metadata: + name: cfroutesync + namespace: #@ data.values.systemNamespace +spec: + selector: #@ labels() + ports: + - port: #@ data.values.service.externalPort + targetPort: 8080 +--- +apiVersion: "apps.cloudfoundry.org/v1alpha1" +kind: RouteBulkSync +metadata: + name: route-bulk-sync + namespace: #@ data.values.workloadsNamespace +spec: + selector: + matchLabels: + cloudfoundry.org/route-bulk-sync: "true" + template: + metadata: + labels: + cloudfoundry.org/route-bulk-sync: "true" diff --git a/examples/ignore/vendor/install/ytt/networking/templates/istio-gateway.yaml b/examples/ignore/vendor/install/ytt/networking/templates/istio-gateway.yaml new file mode 100644 index 00000000..59529794 --- /dev/null +++ b/examples/ignore/vendor/install/ytt/networking/templates/istio-gateway.yaml @@ -0,0 +1,17 @@ +#@ load("@ytt:data", "data") +--- +apiVersion: networking.istio.io/v1alpha3 +kind: Gateway +metadata: + name: istio-ingress + namespace: #@ data.values.workloadsNamespace +spec: + selector: + istio: ingressgateway + servers: + - hosts: + - '*' + port: + name: http + number: #@ data.values.service.externalPort + protocol: HTTP diff --git a/examples/ignore/vendor/install/ytt/networking/values.yaml b/examples/ignore/vendor/install/ytt/networking/values.yaml new file mode 100644 index 00000000..8b7db450 --- /dev/null +++ b/examples/ignore/vendor/install/ytt/networking/values.yaml @@ -0,0 +1,21 @@ +#@data/values +--- +#! Default values for cf-k8s-networking. +#! This is a YTT-formatted file. + +systemNamespace: cf-system +workloadsNamespace: cf-workloads + +cfroutesync: + image: gcr.io/cf-networking-images/cf-k8s-networking/cfroutesync:latest + + ccCA: 'base64_encoded_cloud_controller_ca' + ccBaseURL: 'https://api.example.com' + uaaCA: 'base64_encoded_uaa_ca' + uaaBaseURL: 'https://uaa.example.com' + clientName: 'test' + clientSecret: 'base64_encoded_uaaClientSecret' + eiriniPodLabelPrefix: 'eiriniPodLabelPrefix' + +service: + externalPort: 80 diff --git a/pkg/vendir/config/config.go b/pkg/vendir/config/config.go index 7430c363..e217c735 100644 --- a/pkg/vendir/config/config.go +++ b/pkg/vendir/config/config.go @@ -168,6 +168,7 @@ func (c Config) UseDirectory(path, dirPath string) error { Directory: &DirectoryContentsDirectory{Path: dirPath}, IncludePaths: con.IncludePaths, ExcludePaths: con.ExcludePaths, + IgnorePaths: con.IgnorePaths, LegalPaths: con.LegalPaths, } dir.Contents[j] = newCon diff --git a/pkg/vendir/config/directory.go b/pkg/vendir/config/directory.go index 5693f10e..ad1d090c 100644 --- a/pkg/vendir/config/directory.go +++ b/pkg/vendir/config/directory.go @@ -44,6 +44,7 @@ type DirectoryContents struct { IncludePaths []string `json:"includePaths,omitempty"` ExcludePaths []string `json:"excludePaths,omitempty"` + IgnorePaths []string `json:"ignorePaths,omitempty"` // By default LICENSE/LICENCE/NOTICE/COPYRIGHT files are kept LegalPaths []string `json:"legalPaths,omitempty"` diff --git a/pkg/vendir/directory/directory.go b/pkg/vendir/directory/directory.go index d827199f..0ef61774 100644 --- a/pkg/vendir/directory/directory.go +++ b/pkg/vendir/directory/directory.go @@ -180,6 +180,12 @@ func (d *Directory) Sync(syncOpts SyncOpts) (ctlconf.LockDirectory, error) { } } + // Copy files from current source if values are supposed to be ignored + err = stagingDir.CopyExistingFiles(d.opts.Path, stagingDstPath, contents.IgnorePaths) + if err != nil { + return lockConfig, fmt.Errorf("Copying existing content to staging '%s': %s", d.opts.Path, err) + } + lockConfig.Contents = append(lockConfig.Contents, lockDirContents) } diff --git a/pkg/vendir/directory/staging_dir.go b/pkg/vendir/directory/staging_dir.go index 75749d0a..dbc9e183 100644 --- a/pkg/vendir/directory/staging_dir.go +++ b/pkg/vendir/directory/staging_dir.go @@ -5,10 +5,13 @@ package directory import ( "fmt" + "io" "io/ioutil" "os" "path/filepath" + "strings" + "github.com/bmatcuk/doublestar" ctlfetch "github.com/vmware-tanzu/carvel-vendir/pkg/vendir/fetch" ) @@ -58,6 +61,76 @@ func (d StagingDir) NewChild(path string) (string, error) { return childPath, nil } +func (d StagingDir) CopyExistingFiles(rootDir string, stagingPath string, ignorePaths []string) error { + if len(ignorePaths) == 0 { + return nil + } + + // Create reference point from staging path to root + rootPath := strings.Replace(stagingPath, d.stagingDir, rootDir, 1) + + if _, err := os.Stat(rootPath); os.IsNotExist(err) { + return nil // Path does not exist so there is nothing to copy + } + + var ip []string + for _, ignorePath := range ignorePaths { + ip = append(ip, filepath.Join(rootPath, ignorePath)) // Prefix ignore glob with destination path + } + + // Consider WalkDir in the future for efficiency (Go 1.16) + // Walk root path above to determine files that can be ignored + err := filepath.Walk(rootPath, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + // Verify that the path should be ignored + ignored, err := isPathIgnored(path, ip) + if err != nil { + return err + } + + if !ignored { + return nil + } + + stagingPath := strings.Replace(path, rootPath, stagingPath, 1) // Preserve structure from destination to staging + + // Ensure that the directories exist in the staging directory + stagingDir := filepath.Dir(stagingPath) + err = os.MkdirAll(stagingDir, 0700) + if err != nil { + return fmt.Errorf("Unable to create staging directory '%s': %s", stagingDir, err) + } + + // Move the file to the staging directory + err = copy(path, stagingPath) + if err != nil { + return fmt.Errorf("Moving source file '%s' to staging location '%s': %s", path, stagingPath, err) + } + return nil + }) + if err == os.ErrNotExist { + return nil + } + return err +} + +func isPathIgnored(path string, ignorePaths []string) (bool, error) { + + for _, ip := range ignorePaths { + ok, err := doublestar.PathMatch(ip, path) + if err != nil { + return false, err + } + if ok { + return true, nil + } + } + return false, nil +} + func (d StagingDir) Replace(path string) error { err := os.RemoveAll(path) if err != nil { @@ -121,3 +194,34 @@ func (d StagingTempArea) NewTempDir(name string) (string, error) { func (d StagingTempArea) NewTempFile(pattern string) (*os.File, error) { return ioutil.TempFile(d.path, pattern) } + +func copy(src, dst string) error { + sourceFileStat, err := os.Stat(src) + if err != nil { + return fmt.Errorf("Unable to read file info: %s", src) + } + + if !sourceFileStat.Mode().IsRegular() { + return fmt.Errorf("%s is not a regular file", src) + } + + srcFile, err := os.Open(src) + if err != nil { + return fmt.Errorf("Unable to open file: %s", src) + } + defer srcFile.Close() + + dstFile, err := os.Create(dst) + if err != nil { + return fmt.Errorf("Unable to create destination file: %s", dst) + } + + defer dstFile.Close() + + _, err = io.Copy(dstFile, srcFile) + if err != nil { + return fmt.Errorf("Copying into dst file: %s", err) + } + + return nil +} diff --git a/test/e2e/example.go b/test/e2e/example.go index fbad810a..4d78cd6a 100644 --- a/test/e2e/example.go +++ b/test/e2e/example.go @@ -13,6 +13,7 @@ type example struct { Name string Env []string OnlyLocked bool + SkipRemove bool } func (et example) Check(t *testing.T) { @@ -43,9 +44,11 @@ func (et example) check(t *testing.T, vendir Vendir) error { } // remove all vendored bits - err = os.RemoveAll(vendorPath) - if err != nil { - return fmt.Errorf("Expected no err for remove all") + if !et.SkipRemove { + err = os.RemoveAll(vendorPath) + if err != nil { + return fmt.Errorf("Expected no err for remove all") + } } if !et.OnlyLocked { diff --git a/test/e2e/examples_dir_test.go b/test/e2e/examples_dir_test.go index ead2e4c9..99008d9a 100644 --- a/test/e2e/examples_dir_test.go +++ b/test/e2e/examples_dir_test.go @@ -14,6 +14,7 @@ func TestExamplesDir(t *testing.T) { _ = env.Helm3Binary tests := []example{ + {Name: "git"}, {Name: "http"}, {Name: "image"}, @@ -26,6 +27,7 @@ func TestExamplesDir(t *testing.T) { {Name: "locked", OnlyLocked: true, Env: []string{"VENDIR_HELM_BINARY=" + env.Helm3Binary}}, {Name: "new-root-path"}, {Name: "versionselection"}, + {Name: "ignore", SkipRemove: true}, } for _, test := range tests {