diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..133bec2 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,9 @@ +root = true + +[*] +end_of_line = lf +insert_final_newline = true + +[*.{go}] +indent_style = tab +indent_size = 4 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4426d96 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +/filehitch +/config.yaml +/.vscode/ +__debug_bin* diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 0000000..06d57c3 --- /dev/null +++ b/README.md @@ -0,0 +1,141 @@ +# Filehitch + +> Automate remote file synchronization by monitoring changes in HTTP and S3 resources based on a specified cron interval. Decrypt and trigger commands as needed. + +[![Github release version](https://img.shields.io/github/v/release/surfshark/filehitch.svg?include_prereleases)](https://github.com/surfshark/filehitch/releases/latest) +[![Go report](https://goreportcard.com/badge/github.com/surfshark/filehitch)](https://goreportcard.com/report/github.com/surfshark/filehitch) +[![GoDoc](https://godoc.org/github.com/surfshark/filehitch?status.svg)](https://godoc.org/github.com/surfshark/filehitch) +[![License](https://img.shields.io/github/license/surfshark/filehitch.svg)](https://github.com/surfshark/filehitch/blob/master/LICENSE) +[![Code with hearth by Stnby](https://img.shields.io/badge/%3C%2F%3E%20with%20%E2%99%A5%20by-Stnby-ff1414.svg)](https://github.com/stnby) + +## Install +To install the latest version of Filehitch from sources, run: +```sh +go install github.com/surfshark/filehitch/cmd/filehitch@latest +``` + +To install [Secure IO](https://github.com/minio/sio#readme) file encryption tool from sources, run: +```sh +go install github.com/minio/sio/cmd/ncrypt@latest +``` + +## Features +* **Cron Expression Scheduling**: Set up automated monitoring intervals using precise cron expressions, including seconds, minutes, hours, days of the month, months, days of the week, and multiple timezones support. +* **HTTP and S3 Resource Monitoring**: Monitor remote files via HTTP endpoints and S3 resources, ensuring you stay updated with the latest changes. +* **Secure Encryption**: Utilize the robust ChaCha20-Poly1305 algorithm for seamless decryption of encrypted resources, ensuring secure synchronization and access to sensitive data. +* **Customizable Command Triggers**: Execute custom commands of your choice when changes in remote files are detected, enabling flexible automation. +* **Flexible Configuration**: Intuitive YAML configuration for easy setup and maintenance. +* **User-Friendly Setup**: Intuitive YAML configuration allows for easy and efficient setup and maintenance of the synchronization process. + +## Configuration +```yaml +jobs: + # Define a job named "HTTP example job" + - name: HTTP example job + schedule: + # Set the cron expression for the job to run every hour + expression: "0 0 * * * *" + # Specify the timezone for scheduling (America/New_York) + timezone: America/New_York + # Introduce a jitter of up to 10 minutes to prevent synchronized execution + jitter: 600 + resource: + # Specify the resource type as HTTP for remote file monitoring + type: http + http: + # Specify the URL of the remote file + url: https://example.com/your-remote-file.txt + # Specify the HTTP method (GET, POST, etc.) + method: "GET" + # Specify custom headers if needed + headers: + User-Agent: + - "MyApp/1.0" + Authorization: + - "Bearer YOUR_TOKEN" + # Specify the expected HTTP response code + expect: + code: 200 + # Set a timeout for the HTTP request + timeout: 30s + # Optionally include a request body, use base64 encoded string + body: SGVsbG8sIFdvcmxkIQo= + # Set the maximum number of attempts + max_attempts: 3 + file: + # Specify the local file path to save the downloaded file + path: "/home/alice/your-other-file-path.txt" + permissions: + # Set file permissions to read and write for the owner only + mode: "0600" + # Specify the owner of the file + owner: "alice" + # Specify the group of the file + group: "alice" + trigger: + # Define the command to be triggered upon file change detection + command: ["/home/alice/scripts/run.sh", "--input", "your-other-file-path.txt"] + # Specify the user for executing the trigger command + user: "alice" + # Specify the group for executing the trigger command + group: "alice" + # Set the current working directory for the trigger command + cwd: "/home/alice" + + # Define a job named "S3 example job" + - name: S3 example job + schedule: + # Set the cron expression for the job to run every 30 seconds + expression: "*/30 * * * * *" + # Specify the timezone for scheduling (Europe/Amsterdam) + timezone: Europe/Amsterdam + # Introduce a jitter of up to 5 seconds to prevent synchronized execution + jitter: 5 + resource: + # Specify the resource type as S3 for remote file monitoring + type: s3 + s3: + # Specify the S3 endpoint (for AWS, use s3.amazonaws.com) + endpoint: s3.amazonaws.com + # Specify the AWS region for the S3 bucket + region: your-aws-region + # Specify your AWS access key ID + access_key_id: YOUR_AWS_ACCESS_KEY_ID + # Specify your AWS secret access key + secret_access_key: YOUR_AWS_SECRET_ACCESS_KEY + # Specify the name of the S3 bucket + bucket: my-s3-bucket + # Specify the path to the object within the bucket + object: your-file-path.txt.enc + decryption: + # Specify the decryption password only if the object is encrypted + password: "YourDecryptionPassword" + file: + # Specify the local file path to save the downloaded file + path: "/home/alice/your-file-path.txt" + permissions: + # Set file permissions to read and write for the owner only + mode: "0600" + # Specify the owner of the file + owner: "alice" + # Specify the group of the file + group: "alice" + trigger: + # Define the command to be triggered upon file change detection + command: ["/home/alice/scripts/run.sh", "--input", "your-file-path.txt"] + # Specify the user for executing the trigger command + user: "alice" + # Specify the group for executing the trigger command + group: "alice" + # Set the current working directory for the trigger command + cwd: "/home/alice" +``` + +## Encrypting a file +Make sure [Secure IO](https://github.com/minio/sio#readme) file encryption tool is installed and run: +```sh +ncrypt -cipher C20P1305 your-file.txt > your-file.txt.enc +``` + +## License +This project is licensed under the Apache License, Version 2.0 - see the [LICENSE](https://github.com/surfshark/filehitch/blob/master/LICENSE) file for details. diff --git a/cmd/filehitch/main.go b/cmd/filehitch/main.go new file mode 100644 index 0000000..20e92cb --- /dev/null +++ b/cmd/filehitch/main.go @@ -0,0 +1,128 @@ +// Copyright 2023 Laurynas Četyrkinas +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "encoding/base64" + "flag" + "fmt" + "io/fs" + "strconv" + "time" + + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" + + "github.com/surfshark/filehitch/internal/config" + "github.com/surfshark/filehitch/pkg/filehitch" +) + +func main() { + zerolog.TimeFieldFormat = zerolog.TimeFormatUnix + //log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr, TimeFormat: time.RFC3339}) + debug := flag.Bool("debug", false, "sets log level to debug") + configFile := flag.String("config", "config.yaml", "Configuration file") + flag.Parse() + zerolog.SetGlobalLevel(zerolog.InfoLevel) + if *debug { + zerolog.SetGlobalLevel(zerolog.DebugLevel) + } + cfg, err := config.LoadConfigFile(*configFile) + if err != nil { + log.Error().Err(err).Msg("Failed to load config file") + return + } + jobs := make([]filehitch.Job, len(cfg.Jobs)) + for i, jobCfg := range cfg.Jobs { + jobs[i] = filehitch.Job{ + Name: jobCfg.Name, + Schedule: filehitch.Schedule{ + Expression: jobCfg.Schedule.Expression, + Jitter: jobCfg.Schedule.Jitter, + }, + Resource: filehitch.Resource{ + Type: jobCfg.Resource.Type, + Decryption: filehitch.Decryption{ + Password: []byte(jobCfg.Resource.Decryption.Password), + }, + }, + File: filehitch.File{ + Path: jobCfg.File.Path, + Permissions: filehitch.Permissions{ + Owner: jobCfg.File.Permissions.Owner, + Group: jobCfg.File.Permissions.Group, + }, + }, + Trigger: filehitch.Trigger{ + Command: jobCfg.Trigger.Command, + User: jobCfg.Trigger.User, + Group: jobCfg.Trigger.Group, + CWD: jobCfg.Trigger.CWD, + }, + } + jobs[i].Schedule.Timezone, err = time.LoadLocation(jobCfg.Schedule.Timezone) + if err != nil { + log.Error().Err(err).Str("job", jobs[i].Name).Msg("Failed to load timezone") + return + } + switch jobs[i].Resource.Type { + case "http": + jobs[i].Resource.HTTP = &filehitch.HTTPResource{ + URL: jobCfg.Resource.HTTP.URL, + Method: jobCfg.Resource.HTTP.Method, + Headers: jobCfg.Resource.HTTP.Headers, + Expect: filehitch.Expect{ + Code: jobCfg.Resource.HTTP.Expect.Code, + }, + Timeout: jobCfg.Resource.HTTP.Timeout, + MaxAttempts: jobCfg.Resource.HTTP.MaxAttempts, + } + jobs[i].Resource.HTTP.Body, err = base64.StdEncoding.DecodeString(jobCfg.Resource.HTTP.Body) + if err != nil { + log.Error().Err(err).Str("job", jobs[i].Name).Msg("Failed to decode base64 HTTP resource request body") + return + } + case "s3": + jobs[i].Resource.S3 = &filehitch.S3Resource{ + Endpoint: jobCfg.Resource.S3.Endpoint, + AccessKeyID: jobCfg.Resource.S3.AccessKeyID, + SecretAccessKey: jobCfg.Resource.S3.SecretAccessKey, + Bucket: jobCfg.Resource.S3.Bucket, + SSL: jobCfg.Resource.S3.SSL, + Region: jobCfg.Resource.S3.Region, + Object: jobCfg.Resource.S3.Object, + // TODO: Implement timeout and max attempts. + //Timeout: jobCfg.Resource.S3.Timeout, + //MaxAttempts: jobCfg.Resource.S3.MaxAttempts, + } + } + jobs[i].File.Permissions.Mode, err = stringToFileMode(jobCfg.File.Permissions.Mode) + if err != nil { + log.Error().Err(err).Str("job", jobs[i].Name).Msg("Failed to convert string to file mode") + return + } + } + cfg = &config.Config{} + filehitch.ScheduleJobs(jobs) + select {} +} + +func stringToFileMode(str string) (fs.FileMode, error) { + modeInt, err := strconv.ParseUint(str, 8, 32) + if err != nil { + return 0, fmt.Errorf("failed to convert string to file mode: %w", err) + } + return fs.FileMode(modeInt), nil +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..d975ca6 --- /dev/null +++ b/go.mod @@ -0,0 +1,35 @@ +module github.com/surfshark/filehitch + +go 1.20 + +require ( + github.com/creasty/defaults v1.7.0 + github.com/go-co-op/gocron v1.31.1 + github.com/minio/minio-go/v7 v7.0.61 + github.com/minio/sio v0.3.1 + github.com/rs/zerolog v1.30.0 + golang.org/x/crypto v0.12.0 + gopkg.in/yaml.v3 v3.0.1 +) + +require ( + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/google/uuid v1.3.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.16.7 // indirect + github.com/klauspost/cpuid/v2 v2.2.5 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/minio/sha256-simd v1.0.1 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/robfig/cron/v3 v3.0.1 // indirect + github.com/rs/xid v1.5.0 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect + go.uber.org/atomic v1.11.0 // indirect + golang.org/x/net v0.14.0 // indirect + golang.org/x/sys v0.11.0 // indirect + golang.org/x/text v0.12.0 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..2346209 --- /dev/null +++ b/go.sum @@ -0,0 +1,101 @@ +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/creasty/defaults v1.7.0 h1:eNdqZvc5B509z18lD8yc212CAqJNvfT1Jq6L8WowdBA= +github.com/creasty/defaults v1.7.0/go.mod h1:iGzKe6pbEHnpMPtfDXZEr0NVxWnPTjb1bbDy08fPzYM= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/go-co-op/gocron v1.31.1 h1:LZAuBlU0t3SPGUMJGhrJ6VuCc3CsrYzkzicygvVWlfA= +github.com/go-co-op/gocron v1.31.1/go.mod h1:39f6KNSGVOU1LO/ZOoZfcSxwlsJDQOKSu8erN0SH48Y= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= +github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= +github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.61 h1:87c+x8J3jxQ5VUGimV9oHdpjsAvy3fhneEBKuoKEVUI= +github.com/minio/minio-go/v7 v7.0.61/go.mod h1:BTu8FcrEw+HidY0zd/0eny43QnVNkXRPXrLXFuQBHXg= +github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= +github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= +github.com/minio/sio v0.3.1 h1:d59r5RTHb1OsQaSl1EaTWurzMMDRLA5fgNmjzD4eVu4= +github.com/minio/sio v0.3.1/go.mod h1:S0ovgVgc+sTlQyhiXA1ppBLv7REM7TYi5yyq2qL/Y6o= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.8.1 h1:geMPLpDpQOgVyCg5z5GoRwLHepNdb71NXb67XFkP+Eg= +github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o= +github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= +github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/zerolog v1.30.0 h1:SymVODrcRsaRaSInD9yQtKbtWqwsfoPcRff/oRXLj4c= +github.com/rs/zerolog v1.30.0/go.mod h1:/tk+P47gFdPXq4QYjvCmT5/Gsug2nagsFWBWhAiSi1w= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= +golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= +golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= +golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= +golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/config/config.go b/internal/config/config.go new file mode 100644 index 0000000..f6c1151 --- /dev/null +++ b/internal/config/config.go @@ -0,0 +1,115 @@ +// Copyright 2023 Laurynas Četyrkinas +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "os" + "time" + + "github.com/creasty/defaults" + "gopkg.in/yaml.v3" +) + +type Config struct { + Jobs []Job `yaml:"jobs"` +} + +type Job struct { + Name string `yaml:"name"` + Schedule Schedule `yaml:"schedule"` + Resource Resource `yaml:"resource"` + File File `yaml:"file"` + Trigger Trigger `yaml:"trigger"` +} + +type Schedule struct { + Expression string `yaml:"expression"` + Timezone string `yaml:"timezone" default:"Local"` + Jitter int `yaml:"jitter" default:"5"` +} + +type Resource struct { + Type string `yaml:"type"` + Decryption Decryption `yaml:"decryption"` + HTTP *HTTPResource `yaml:"http"` + S3 *S3Resource `yaml:"s3"` +} + +type HTTPResource struct { + URL string `yaml:"url"` + Method string `yaml:"method"` + Headers map[string][]string `yaml:"headers"` + Expect Expect `yaml:"expect"` + Timeout time.Duration `yaml:"timeout"` + MaxAttempts int `yaml:"max_attempts" default:"3"` + Body string `yaml:"body"` +} + +type S3Resource struct { + Endpoint string `yaml:"endpoint"` + AccessKeyID string `yaml:"access_key_id"` + SecretAccessKey string `yaml:"secret_access_key"` + Bucket string `yaml:"bucket"` + Region string `yaml:"region"` + SSL bool `yaml:"ssl" default:"true"` + Object string `yaml:"object"` +} + +type Expect struct { + Code int `yaml:"code"` + Body string `yaml:"body"` +} + +type Decryption struct { + Password string `yaml:"password"` +} + +type File struct { + Path string `yaml:"path"` + Permissions Permissions `yaml:"permissions"` +} + +type Permissions struct { + Mode string `yaml:"mode" default:"0644"` + Owner string `yaml:"owner" default:"root"` + Group string `yaml:"group" default:"root"` +} + +type Trigger struct { + Command []string `yaml:"command"` + User string `yaml:"user" default:"root"` + Group string `yaml:"group" default:"root"` + CWD string `yaml:"cwd"` +} + +func LoadConfigFile(filename string) (*Config, error) { + data, err := os.ReadFile(filename) + if err != nil { + return nil, err + } + + var cfg Config + err = yaml.Unmarshal(data, &cfg) + if err != nil { + return nil, err + } + + err = defaults.Set(&cfg) + if err != nil { + return nil, err + } + + return &cfg, nil +} diff --git a/pkg/filehitch/filehitch.go b/pkg/filehitch/filehitch.go new file mode 100644 index 0000000..84b4c0c --- /dev/null +++ b/pkg/filehitch/filehitch.go @@ -0,0 +1,273 @@ +// Copyright 2023 Laurynas Četyrkinas +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package filehitch + +import ( + "bytes" + "crypto/sha512" + "fmt" + "io" + "io/fs" + "math/rand" + "os" + "os/exec" + "os/user" + "strconv" + "syscall" + "time" + + "github.com/go-co-op/gocron" + "github.com/minio/sio" + "github.com/rs/zerolog/log" + "golang.org/x/crypto/scrypt" +) + +type Job struct { + Name string + Schedule Schedule + Resource Resource + File File + Trigger Trigger +} + +type Schedule struct { + Expression string + Timezone *time.Location + Jitter int +} + +type Resource struct { + Type string + Decryption Decryption + HTTP *HTTPResource + S3 *S3Resource +} + +type Decryption struct { + Password []byte +} + +type File struct { + Path string + Permissions Permissions +} + +type Permissions struct { + Mode fs.FileMode + Owner string + Group string +} + +type Trigger struct { + Command []string + User string + Group string + CWD string +} + +func ScheduleJobs(jobs []Job) { + // Store schedulers in a map with timezone strings as keys, + // as each scheduler has its own timezone setting. + scheds := make(map[string]*gocron.Scheduler) + for i := 0; i < len(jobs); i++ { + job := &jobs[i] + tzname := job.Schedule.Timezone.String() + sched, ok := scheds[tzname] + if !ok { + sched = gocron.NewScheduler(job.Schedule.Timezone) + scheds[tzname] = sched + log.Debug().Str("job", job.Name).Str("scheduler", tzname).Msg("Created scheduler") + } + sched.CronWithSeconds(job.Schedule.Expression).Do(job.Run) + log.Debug().Str("job", job.Name).Str("scheduler", tzname).Str("expression", job.Schedule.Expression).Msg("Added cron job to scheduler") + } + for tzname, sched := range scheds { + sched.StartAsync() + log.Debug().Str("scheduler", tzname).Msg("Started scheduler") + } +} + +func (job *Job) Run() { + if job.Schedule.Jitter > 0 { + jitter := time.Millisecond * time.Duration(rand.Intn(int(job.Schedule.Jitter)*1000)) + log.Debug().Str("job", job.Name).Dur("jitter", jitter).Msg("Delaying due to jitter") + time.Sleep(jitter) + } + log.Info().Str("job", job.Name).Msg("Starting job") + var ( + updated bool + err error + ) + switch job.Resource.Type { + case "http": + updated, err = job.HandleHTTPResource() + case "s3": + updated, err = job.HandleS3Resource() + } + if err != nil { + log.Error().Str("job", job.Name).Err(err).Msg("Failed job") + return + } + if !updated { + log.Info().Str("job", job.Name).Msg("Finished job, file not updated") + return + } + if len(job.Trigger.Command) > 0 { + err = job.ExecuteTrigger() + if err != nil { + log.Error().Str("job", job.Name).Err(err).Msg("Failed to execute trigger") + } + } + log.Info().Str("job", job.Name).Msg("Finished job, file updated") +} + +func (job *Job) UpdateFile(src io.ReadCloser) (changed bool, err error) { + tempFile, err := CreateTempFile() + if err != nil { + return + } + name := tempFile.Name() + log.Debug().Str("job", job.Name).Str("file", name).Msg("Created temporary file") + defer tempFile.Close() + + var tempHash []byte + if len(job.Resource.Decryption.Password) > 0 { + tempHash, err = DecryptWriteToFileAndChecksum(tempFile, src, job.Resource.Decryption.Password) + } else { + tempHash, err = WriteToFileAndChecksum(tempFile, src) + } + if err != nil { + return + } + currHash, err := job.CalculateFileChecksum() + if err != nil { + if os.IsNotExist(err) { + err = job.FinalizePlacement(tempFile) + if err == nil { + changed = true + } + return + } + } + if !bytes.Equal(tempHash, currHash) { + err = job.FinalizePlacement(tempFile) + if err == nil { + changed = true + } + return + } + err = os.Remove(name) + if err == nil { + log.Debug().Str("job", job.Name).Str("file", name).Msg("Removed temporary file") + } + return +} + +func CreateTempFile() (file *os.File, err error) { + file, err = os.CreateTemp("", "rrchanged-") + if err != nil { + return + } + return file, os.Chmod(file.Name(), 0600) +} + +func DecryptWriteToFileAndChecksum(file *os.File, src io.Reader, password []byte) (sum []byte, err error) { + salt := make([]byte, 32) + if _, err = io.ReadFull(src, salt); err != nil { + err = fmt.Errorf("failed to read salt: %w", err) + return + } + key, err := scrypt.Key(password, salt, 32768, 16, 1, 32) + if err != nil { + err = fmt.Errorf("failed to derive key from password and salt: %w", err) + return + } + cfg := sio.Config{Key: key, CipherSuites: []byte{sio.CHACHA20_POLY1305}} + h := sha512.New() + _, err = sio.Decrypt(io.MultiWriter(file, h), src, cfg) + if err != nil { + err = fmt.Errorf("failed to decrypt: %w", err) + return + } + sum = h.Sum(nil) + return +} + +func WriteToFileAndChecksum(file *os.File, src io.Reader) (sum []byte, err error) { + h := sha512.New() + _, err = io.Copy(io.MultiWriter(file, h), src) + sum = h.Sum(nil) + return +} + +func (job *Job) CalculateFileChecksum() (sum []byte, err error) { + f, err := os.Open(job.File.Path) + if err != nil { + return + } + defer f.Close() + h := sha512.New() + _, err = io.Copy(h, f) + sum = h.Sum(nil) + return +} + +func (job *Job) FinalizePlacement(tempFile *os.File) (err error) { + name := tempFile.Name() + tempFile.Close() + err = os.Rename(name, job.File.Path) + if err != nil { + return + } + u, err := user.Lookup(job.File.Permissions.Owner) + if err != nil { + return + } + g, err := user.LookupGroup(job.File.Permissions.Group) + if err != nil { + return + } + uid, _ := strconv.Atoi(u.Uid) + gid, _ := strconv.Atoi(g.Gid) + err = os.Chown(job.File.Path, uid, gid) + if err != nil { + return + } + return os.Chmod(job.File.Path, job.File.Permissions.Mode) +} + +func (job *Job) ExecuteTrigger() (err error) { + cmd := exec.Command(job.Trigger.Command[0], job.Trigger.Command[1:]...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Dir = job.Trigger.CWD + u, err := user.Lookup(job.Trigger.User) + if err != nil { + return + } + g, err := user.LookupGroup(job.Trigger.Group) + if err != nil { + return + } + uid, _ := strconv.Atoi(u.Uid) + gid, _ := strconv.Atoi(g.Gid) + log.Debug().Str("user", job.Trigger.User).Str("group", job.Trigger.Group).Str("cwd", job.Trigger.CWD).Strs("command", job.Trigger.Command).Msg("Executing trigger command") + cmd.SysProcAttr = &syscall.SysProcAttr{ + Credential: &syscall.Credential{ + Uid: uint32(uid), Gid: uint32(gid), + }, + } + return cmd.Run() +} diff --git a/pkg/filehitch/http_resource.go b/pkg/filehitch/http_resource.go new file mode 100644 index 0000000..7919cb5 --- /dev/null +++ b/pkg/filehitch/http_resource.go @@ -0,0 +1,109 @@ +// Copyright 2023 Laurynas Četyrkinas +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package filehitch + +import ( + "bytes" + "fmt" + "net/http" + "time" + + "github.com/rs/zerolog/log" +) + +type HTTPResource struct { + URL string + Method string + Headers map[string][]string + Expect Expect + Timeout time.Duration + Body []byte + MaxAttempts int + headerIfNoneMatch string + headerIfModifiedSince string +} + +type Expect struct { + Code int +} + +func (job *Job) HandleHTTPResource() (changed bool, err error) { + c := &http.Client{ + Timeout: job.Resource.HTTP.Timeout, + } + for attempt := 1; attempt <= job.Resource.HTTP.MaxAttempts; attempt++ { + var req *http.Request + req, err = http.NewRequest(job.Resource.HTTP.Method, job.Resource.HTTP.URL, bytes.NewBuffer(job.Resource.HTTP.Body)) + if err != nil { + return + } + if job.Resource.HTTP.headerIfNoneMatch != "" { + req.Header.Add("If-None-Match", job.Resource.HTTP.headerIfNoneMatch) + } + // If-Modified-Since can only be used with a GET or HEAD. + // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/If-Modified-Since + if job.Resource.HTTP.Method == http.MethodGet || job.Resource.HTTP.Method == http.MethodHead { + if job.Resource.HTTP.headerIfModifiedSince != "" { + req.Header.Add("If-Modified-Since", job.Resource.HTTP.headerIfModifiedSince) + } + } + job.Resource.HTTP.AddHeaders(req) + var resp *http.Response + resp, err = c.Do(req) + if err != nil { + if attempt < job.Resource.HTTP.MaxAttempts { + log.Warn().Str("job", job.Name).Int("attempt", attempt).Int("max_attempts", job.Resource.HTTP.MaxAttempts).Err(err).Msg("Failed HTTP request, retrying") + time.Sleep(time.Second) + continue + } + log.Error().Str("job", job.Name).Int("attempt", attempt).Int("max_attempts", job.Resource.HTTP.MaxAttempts).Err(err).Msg("Failed HTTP request") + return + } + defer resp.Body.Close() + if resp.StatusCode == http.StatusNotModified { + return + } + if resp.StatusCode != job.Resource.HTTP.Expect.Code { + err = fmt.Errorf("unexpected HTTP status code %d, expected %d", resp.StatusCode, job.Resource.HTTP.Expect.Code) + if attempt < job.Resource.HTTP.MaxAttempts { + log.Warn().Str("job", job.Name).Int("attempt", attempt).Int("max_attempts", job.Resource.HTTP.MaxAttempts).Err(err).Msg("Unexpected HTTP status code, retyring") + time.Sleep(time.Second) + continue + } + log.Error().Str("job", job.Name).Int("attempt", attempt).Int("max_attempts", job.Resource.HTTP.MaxAttempts).Err(err).Msg("Unexpected HTTP status code") + return + } + changed, err = job.UpdateFile(resp.Body) + if err != nil { + return + } + job.Resource.HTTP.headerIfNoneMatch = resp.Header.Get("ETag") + // If-Modified-Since can only be used with a GET or HEAD. + // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/If-Modified-Since + if job.Resource.HTTP.Method == http.MethodGet || job.Resource.HTTP.Method == http.MethodHead { + job.Resource.HTTP.headerIfModifiedSince = resp.Header.Get("Date") + } + return + } + return +} + +func (resource *HTTPResource) AddHeaders(req *http.Request) { + for key, values := range resource.Headers { + for _, value := range values { + req.Header.Add(key, value) + } + } +} diff --git a/pkg/filehitch/s3_resource.go b/pkg/filehitch/s3_resource.go new file mode 100644 index 0000000..59f24f2 --- /dev/null +++ b/pkg/filehitch/s3_resource.go @@ -0,0 +1,62 @@ +// Copyright 2023 Laurynas Četyrkinas +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package filehitch + +import ( + "context" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +type S3Resource struct { + Endpoint string + AccessKeyID string + SecretAccessKey string + Bucket string + Region string + SSL bool + Object string + objInfoETag string +} + +func (job *Job) HandleS3Resource() (changed bool, err error) { + c, err := minio.New(job.Resource.S3.Endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(job.Resource.S3.AccessKeyID, job.Resource.S3.SecretAccessKey, ""), + Secure: job.Resource.S3.SSL, + }) + if err != nil { + return + } + ctx := context.Background() + objInfo, err := c.StatObject(ctx, job.Resource.S3.Bucket, job.Resource.S3.Object, minio.StatObjectOptions{}) + if err != nil { + return + } + if job.Resource.S3.objInfoETag == objInfo.ETag { + return + } + obj, err := c.GetObject(ctx, job.Resource.S3.Bucket, job.Resource.S3.Object, minio.GetObjectOptions{}) + if err != nil { + return + } + defer obj.Close() + changed, err = job.UpdateFile(obj) + if err != nil { + return + } + job.Resource.S3.objInfoETag = objInfo.ETag + return +}