From d37b6c0e702ac72224ecdda1b53e5588e68c0802 Mon Sep 17 00:00:00 2001 From: Neil Hemming Date: Tue, 20 Jul 2021 00:18:06 +0100 Subject: [PATCH] feat: refactor to support variables, cli params --- .vscode/settings.json | 3 +- CREDITS | 719 +++++++++++++++ build/stdbuild.yml | 33 +- go.mod | 1 + go.sum | 3 + internal/cmd/initcoonfig.yml | 79 +- internal/cmd/launch.go | 40 +- internal/cmd/launch_test.go | 53 ++ pkg/loggee/log.go | 5 + pkg/providers/closerlist.go | 47 + pkg/providers/closerlist_test.go | 253 ++++++ pkg/providers/fileprovider.go | 156 ++++ pkg/providers/fileprovider_test.go | 313 +++++++ pkg/providers/iomodes.go | 52 ++ pkg/providers/iomodes_test.go | 73 ++ pkg/providers/lineprovider.go | 63 ++ pkg/providers/lineprovider_test.go | 45 + pkg/providers/logprovider.go | 33 + pkg/providers/logprovider_test.go | 80 ++ pkg/providers/multipleerrors.go | 33 + pkg/providers/multipleerrors_test.go | 48 + pkg/providers/providers.go | 60 ++ pkg/providers/providers_test.go | 21 + pkg/providers/testdata/append.dat | 1 + pkg/providers/testdata/trunc.dat | 1 + pkg/providers/urlprovider.go | 75 ++ pkg/providers/urlprovider_test.go | 105 +++ pkg/rocket/builtin/fetch.go | 142 ++- pkg/rocket/builtin/redirect.go | 154 ---- pkg/rocket/builtin/run.go | 68 +- pkg/rocket/builtin/runinit_test.go | 23 + pkg/rocket/builtin/template.go | 112 +-- pkg/rocket/builtin/testdata/badfetch.yml | 12 +- pkg/rocket/builtin/testdata/fetch.yml | 6 +- pkg/rocket/builtin/testdata/hello.yml | 11 +- pkg/rocket/builtin/testdata/init_output.yml | 174 ++++ pkg/rocket/builtin/testdata/rungo.yml | 3 +- pkg/rocket/capcomm.go | 441 ++++++++-- pkg/rocket/capcomm_test.go | 918 +++++++++++++++----- pkg/rocket/config.go | 78 +- pkg/rocket/context_test.go | 4 +- pkg/rocket/getters.go | 11 + pkg/rocket/include.go | 86 +- pkg/rocket/io.go | 141 +-- pkg/rocket/io_test.go | 118 --- pkg/rocket/missioncontrol.go | 18 +- pkg/rocket/missioncontrol_test.go | 5 + pkg/rocket/variablewriter.go | 39 + 48 files changed, 3970 insertions(+), 989 deletions(-) create mode 100644 internal/cmd/launch_test.go create mode 100644 pkg/providers/closerlist.go create mode 100644 pkg/providers/closerlist_test.go create mode 100644 pkg/providers/fileprovider.go create mode 100644 pkg/providers/fileprovider_test.go create mode 100644 pkg/providers/iomodes.go create mode 100644 pkg/providers/iomodes_test.go create mode 100644 pkg/providers/lineprovider.go create mode 100644 pkg/providers/lineprovider_test.go create mode 100644 pkg/providers/logprovider.go create mode 100644 pkg/providers/logprovider_test.go create mode 100644 pkg/providers/multipleerrors.go create mode 100644 pkg/providers/multipleerrors_test.go create mode 100644 pkg/providers/providers.go create mode 100644 pkg/providers/providers_test.go create mode 100644 pkg/providers/testdata/append.dat create mode 100644 pkg/providers/testdata/trunc.dat create mode 100644 pkg/providers/urlprovider.go create mode 100644 pkg/providers/urlprovider_test.go delete mode 100644 pkg/rocket/builtin/redirect.go create mode 100644 pkg/rocket/builtin/runinit_test.go create mode 100644 pkg/rocket/builtin/testdata/init_output.yml delete mode 100644 pkg/rocket/io_test.go create mode 100644 pkg/rocket/variablewriter.go diff --git a/.vscode/settings.json b/.vscode/settings.json index 6026b19..9926599 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -12,6 +12,7 @@ "ironment", "logcli", "nehemming", - "nolint" + "nolint", + "testdata" ] } \ No newline at end of file diff --git a/CREDITS b/CREDITS index 0463887..9a5193d 100644 --- a/CREDITS +++ b/CREDITS @@ -170,6 +170,725 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================ +github.com/hashicorp/errwrap +https://github.com/hashicorp/errwrap +---------------------------------------------------------------- +Mozilla Public License, version 2.0 + +1. Definitions + +1.1. “Contributor” + + means each individual or legal entity that creates, contributes to the + creation of, or owns Covered Software. + +1.2. “Contributor Version” + + means the combination of the Contributions of others (if any) used by a + Contributor and that particular Contributor’s Contribution. + +1.3. “Contribution” + + means Covered Software of a particular Contributor. + +1.4. “Covered Software” + + means Source Code Form to which the initial Contributor has attached the + notice in Exhibit A, the Executable Form of such Source Code Form, and + Modifications of such Source Code Form, in each case including portions + thereof. + +1.5. “Incompatible With Secondary Licenses” + means + + a. that the initial Contributor has attached the notice described in + Exhibit B to the Covered Software; or + + b. that the Covered Software was made available under the terms of version + 1.1 or earlier of the License, but not also under the terms of a + Secondary License. + +1.6. “Executable Form” + + means any form of the work other than Source Code Form. + +1.7. “Larger Work” + + means a work that combines Covered Software with other material, in a separate + file or files, that is not Covered Software. + +1.8. “License” + + means this document. + +1.9. “Licensable” + + means having the right to grant, to the maximum extent possible, whether at the + time of the initial grant or subsequently, any and all of the rights conveyed by + this License. + +1.10. “Modifications” + + means any of the following: + + a. any file in Source Code Form that results from an addition to, deletion + from, or modification of the contents of Covered Software; or + + b. any new file in Source Code Form that contains any Covered Software. + +1.11. “Patent Claims” of a Contributor + + means any patent claim(s), including without limitation, method, process, + and apparatus claims, in any patent Licensable by such Contributor that + would be infringed, but for the grant of the License, by the making, + using, selling, offering for sale, having made, import, or transfer of + either its Contributions or its Contributor Version. + +1.12. “Secondary License” + + means either the GNU General Public License, Version 2.0, the GNU Lesser + General Public License, Version 2.1, the GNU Affero General Public + License, Version 3.0, or any later versions of those licenses. + +1.13. “Source Code Form” + + means the form of the work preferred for making modifications. + +1.14. “You” (or “Your”) + + means an individual or a legal entity exercising rights under this + License. For legal entities, “You” includes any entity that controls, is + controlled by, or is under common control with You. For purposes of this + definition, “control” means (a) the power, direct or indirect, to cause + the direction or management of such entity, whether by contract or + otherwise, or (b) ownership of more than fifty percent (50%) of the + outstanding shares or beneficial ownership of such entity. + + +2. License Grants and Conditions + +2.1. Grants + + Each Contributor hereby grants You a world-wide, royalty-free, + non-exclusive license: + + a. under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or as + part of a Larger Work; and + + b. under Patent Claims of such Contributor to make, use, sell, offer for + sale, have made, import, and otherwise transfer either its Contributions + or its Contributor Version. + +2.2. Effective Date + + The licenses granted in Section 2.1 with respect to any Contribution become + effective for each Contribution on the date the Contributor first distributes + such Contribution. + +2.3. Limitations on Grant Scope + + The licenses granted in this Section 2 are the only rights granted under this + License. No additional rights or licenses will be implied from the distribution + or licensing of Covered Software under this License. Notwithstanding Section + 2.1(b) above, no patent license is granted by a Contributor: + + a. for any code that a Contributor has removed from Covered Software; or + + b. for infringements caused by: (i) Your and any other third party’s + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + + c. under Patent Claims infringed by Covered Software in the absence of its + Contributions. + + This License does not grant any rights in the trademarks, service marks, or + logos of any Contributor (except as may be necessary to comply with the + notice requirements in Section 3.4). + +2.4. Subsequent Licenses + + No Contributor makes additional grants as a result of Your choice to + distribute the Covered Software under a subsequent version of this License + (see Section 10.2) or under the terms of a Secondary License (if permitted + under the terms of Section 3.3). + +2.5. Representation + + Each Contributor represents that the Contributor believes its Contributions + are its original creation(s) or it has sufficient rights to grant the + rights to its Contributions conveyed by this License. + +2.6. Fair Use + + This License is not intended to limit any rights You have under applicable + copyright doctrines of fair use, fair dealing, or other equivalents. + +2.7. Conditions + + Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in + Section 2.1. + + +3. Responsibilities + +3.1. Distribution of Source Form + + All distribution of Covered Software in Source Code Form, including any + Modifications that You create or to which You contribute, must be under the + terms of this License. You must inform recipients that the Source Code Form + of the Covered Software is governed by the terms of this License, and how + they can obtain a copy of this License. You may not attempt to alter or + restrict the recipients’ rights in the Source Code Form. + +3.2. Distribution of Executable Form + + If You distribute Covered Software in Executable Form then: + + a. such Covered Software must also be made available in Source Code Form, + as described in Section 3.1, and You must inform recipients of the + Executable Form how they can obtain a copy of such Source Code Form by + reasonable means in a timely manner, at a charge no more than the cost + of distribution to the recipient; and + + b. You may distribute such Executable Form under the terms of this License, + or sublicense it under different terms, provided that the license for + the Executable Form does not attempt to limit or alter the recipients’ + rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + + You may create and distribute a Larger Work under terms of Your choice, + provided that You also comply with the requirements of this License for the + Covered Software. If the Larger Work is a combination of Covered Software + with a work governed by one or more Secondary Licenses, and the Covered + Software is not Incompatible With Secondary Licenses, this License permits + You to additionally distribute such Covered Software under the terms of + such Secondary License(s), so that the recipient of the Larger Work may, at + their option, further distribute the Covered Software under the terms of + either this License or such Secondary License(s). + +3.4. Notices + + You may not remove or alter the substance of any license notices (including + copyright notices, patent notices, disclaimers of warranty, or limitations + of liability) contained within the Source Code Form of the Covered + Software, except that You may alter any license notices to the extent + required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + + You may choose to offer, and to charge a fee for, warranty, support, + indemnity or liability obligations to one or more recipients of Covered + Software. However, You may do so only on Your own behalf, and not on behalf + of any Contributor. You must make it absolutely clear that any such + warranty, support, indemnity, or liability obligation is offered by You + alone, and You hereby agree to indemnify every Contributor for any + liability incurred by such Contributor as a result of warranty, support, + indemnity or liability terms You offer. You may include additional + disclaimers of warranty and limitations of liability specific to any + jurisdiction. + +4. Inability to Comply Due to Statute or Regulation + + If it is impossible for You to comply with any of the terms of this License + with respect to some or all of the Covered Software due to statute, judicial + order, or regulation then You must: (a) comply with the terms of this License + to the maximum extent possible; and (b) describe the limitations and the code + they affect. Such description must be placed in a text file included with all + distributions of the Covered Software under this License. Except to the + extent prohibited by statute or regulation, such description must be + sufficiently detailed for a recipient of ordinary skill to be able to + understand it. + +5. Termination + +5.1. The rights granted under this License will terminate automatically if You + fail to comply with any of its terms. However, if You become compliant, + then the rights granted under this License from a particular Contributor + are reinstated (a) provisionally, unless and until such Contributor + explicitly and finally terminates Your grants, and (b) on an ongoing basis, + if such Contributor fails to notify You of the non-compliance by some + reasonable means prior to 60 days after You have come back into compliance. + Moreover, Your grants from a particular Contributor are reinstated on an + ongoing basis if such Contributor notifies You of the non-compliance by + some reasonable means, this is the first time You have received notice of + non-compliance with this License from such Contributor, and You become + compliant prior to 30 days after Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent + infringement claim (excluding declaratory judgment actions, counter-claims, + and cross-claims) alleging that a Contributor Version directly or + indirectly infringes any patent, then the rights granted to You by any and + all Contributors for the Covered Software under Section 2.1 of this License + shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user + license agreements (excluding distributors and resellers) which have been + validly granted by You or Your distributors under this License prior to + termination shall survive termination. + +6. Disclaimer of Warranty + + Covered Software is provided under this License on an “as is” basis, without + warranty of any kind, either expressed, implied, or statutory, including, + without limitation, warranties that the Covered Software is free of defects, + merchantable, fit for a particular purpose or non-infringing. The entire + risk as to the quality and performance of the Covered Software is with You. + Should any Covered Software prove defective in any respect, You (not any + Contributor) assume the cost of any necessary servicing, repair, or + correction. This disclaimer of warranty constitutes an essential part of this + License. No use of any Covered Software is authorized under this License + except under this disclaimer. + +7. Limitation of Liability + + Under no circumstances and under no legal theory, whether tort (including + negligence), contract, or otherwise, shall any Contributor, or anyone who + distributes Covered Software as permitted above, be liable to You for any + direct, indirect, special, incidental, or consequential damages of any + character including, without limitation, damages for lost profits, loss of + goodwill, work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses, even if such party shall have been + informed of the possibility of such damages. This limitation of liability + shall not apply to liability for death or personal injury resulting from such + party’s negligence to the extent applicable law prohibits such limitation. + Some jurisdictions do not allow the exclusion or limitation of incidental or + consequential damages, so this exclusion and limitation may not apply to You. + +8. Litigation + + Any litigation relating to this License may be brought only in the courts of + a jurisdiction where the defendant maintains its principal place of business + and such litigation shall be governed by laws of that jurisdiction, without + reference to its conflict-of-law provisions. Nothing in this Section shall + prevent a party’s ability to bring cross-claims or counter-claims. + +9. Miscellaneous + + This License represents the complete agreement concerning the subject matter + hereof. If any provision of this License is held to be unenforceable, such + provision shall be reformed only to the extent necessary to make it + enforceable. Any law or regulation which provides that the language of a + contract shall be construed against the drafter shall not be used to construe + this License against a Contributor. + + +10. Versions of the License + +10.1. New Versions + + Mozilla Foundation is the license steward. Except as provided in Section + 10.3, no one other than the license steward has the right to modify or + publish new versions of this License. Each version will be given a + distinguishing version number. + +10.2. Effect of New Versions + + You may distribute the Covered Software under the terms of the version of + the License under which You originally received the Covered Software, or + under the terms of any subsequent version published by the license + steward. + +10.3. Modified Versions + + If you create software not governed by this License, and you want to + create a new license for such software, you may create and use a modified + version of this License if you rename the license and remove any + references to the name of the license steward (except to note that such + modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses + If You choose to distribute Source Code Form that is Incompatible With + Secondary Licenses under the terms of this version of the License, the + notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice + + This Source Code Form is subject to the + terms of the Mozilla Public License, v. + 2.0. If a copy of the MPL was not + distributed with this file, You can + obtain one at + http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular file, then +You may include the notice in a location (such as a LICENSE file in a relevant +directory) where a recipient would be likely to look for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - “Incompatible With Secondary Licenses” Notice + + This Source Code Form is “Incompatible + With Secondary Licenses”, as defined by + the Mozilla Public License, v. 2.0. + + +================================================================ + +github.com/hashicorp/go-multierror +https://github.com/hashicorp/go-multierror +---------------------------------------------------------------- +Mozilla Public License, version 2.0 + +1. Definitions + +1.1. “Contributor” + + means each individual or legal entity that creates, contributes to the + creation of, or owns Covered Software. + +1.2. “Contributor Version” + + means the combination of the Contributions of others (if any) used by a + Contributor and that particular Contributor’s Contribution. + +1.3. “Contribution” + + means Covered Software of a particular Contributor. + +1.4. “Covered Software” + + means Source Code Form to which the initial Contributor has attached the + notice in Exhibit A, the Executable Form of such Source Code Form, and + Modifications of such Source Code Form, in each case including portions + thereof. + +1.5. “Incompatible With Secondary Licenses” + means + + a. that the initial Contributor has attached the notice described in + Exhibit B to the Covered Software; or + + b. that the Covered Software was made available under the terms of version + 1.1 or earlier of the License, but not also under the terms of a + Secondary License. + +1.6. “Executable Form” + + means any form of the work other than Source Code Form. + +1.7. “Larger Work” + + means a work that combines Covered Software with other material, in a separate + file or files, that is not Covered Software. + +1.8. “License” + + means this document. + +1.9. “Licensable” + + means having the right to grant, to the maximum extent possible, whether at the + time of the initial grant or subsequently, any and all of the rights conveyed by + this License. + +1.10. “Modifications” + + means any of the following: + + a. any file in Source Code Form that results from an addition to, deletion + from, or modification of the contents of Covered Software; or + + b. any new file in Source Code Form that contains any Covered Software. + +1.11. “Patent Claims” of a Contributor + + means any patent claim(s), including without limitation, method, process, + and apparatus claims, in any patent Licensable by such Contributor that + would be infringed, but for the grant of the License, by the making, + using, selling, offering for sale, having made, import, or transfer of + either its Contributions or its Contributor Version. + +1.12. “Secondary License” + + means either the GNU General Public License, Version 2.0, the GNU Lesser + General Public License, Version 2.1, the GNU Affero General Public + License, Version 3.0, or any later versions of those licenses. + +1.13. “Source Code Form” + + means the form of the work preferred for making modifications. + +1.14. “You” (or “Your”) + + means an individual or a legal entity exercising rights under this + License. For legal entities, “You” includes any entity that controls, is + controlled by, or is under common control with You. For purposes of this + definition, “control” means (a) the power, direct or indirect, to cause + the direction or management of such entity, whether by contract or + otherwise, or (b) ownership of more than fifty percent (50%) of the + outstanding shares or beneficial ownership of such entity. + + +2. License Grants and Conditions + +2.1. Grants + + Each Contributor hereby grants You a world-wide, royalty-free, + non-exclusive license: + + a. under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or as + part of a Larger Work; and + + b. under Patent Claims of such Contributor to make, use, sell, offer for + sale, have made, import, and otherwise transfer either its Contributions + or its Contributor Version. + +2.2. Effective Date + + The licenses granted in Section 2.1 with respect to any Contribution become + effective for each Contribution on the date the Contributor first distributes + such Contribution. + +2.3. Limitations on Grant Scope + + The licenses granted in this Section 2 are the only rights granted under this + License. No additional rights or licenses will be implied from the distribution + or licensing of Covered Software under this License. Notwithstanding Section + 2.1(b) above, no patent license is granted by a Contributor: + + a. for any code that a Contributor has removed from Covered Software; or + + b. for infringements caused by: (i) Your and any other third party’s + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + + c. under Patent Claims infringed by Covered Software in the absence of its + Contributions. + + This License does not grant any rights in the trademarks, service marks, or + logos of any Contributor (except as may be necessary to comply with the + notice requirements in Section 3.4). + +2.4. Subsequent Licenses + + No Contributor makes additional grants as a result of Your choice to + distribute the Covered Software under a subsequent version of this License + (see Section 10.2) or under the terms of a Secondary License (if permitted + under the terms of Section 3.3). + +2.5. Representation + + Each Contributor represents that the Contributor believes its Contributions + are its original creation(s) or it has sufficient rights to grant the + rights to its Contributions conveyed by this License. + +2.6. Fair Use + + This License is not intended to limit any rights You have under applicable + copyright doctrines of fair use, fair dealing, or other equivalents. + +2.7. Conditions + + Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in + Section 2.1. + + +3. Responsibilities + +3.1. Distribution of Source Form + + All distribution of Covered Software in Source Code Form, including any + Modifications that You create or to which You contribute, must be under the + terms of this License. You must inform recipients that the Source Code Form + of the Covered Software is governed by the terms of this License, and how + they can obtain a copy of this License. You may not attempt to alter or + restrict the recipients’ rights in the Source Code Form. + +3.2. Distribution of Executable Form + + If You distribute Covered Software in Executable Form then: + + a. such Covered Software must also be made available in Source Code Form, + as described in Section 3.1, and You must inform recipients of the + Executable Form how they can obtain a copy of such Source Code Form by + reasonable means in a timely manner, at a charge no more than the cost + of distribution to the recipient; and + + b. You may distribute such Executable Form under the terms of this License, + or sublicense it under different terms, provided that the license for + the Executable Form does not attempt to limit or alter the recipients’ + rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + + You may create and distribute a Larger Work under terms of Your choice, + provided that You also comply with the requirements of this License for the + Covered Software. If the Larger Work is a combination of Covered Software + with a work governed by one or more Secondary Licenses, and the Covered + Software is not Incompatible With Secondary Licenses, this License permits + You to additionally distribute such Covered Software under the terms of + such Secondary License(s), so that the recipient of the Larger Work may, at + their option, further distribute the Covered Software under the terms of + either this License or such Secondary License(s). + +3.4. Notices + + You may not remove or alter the substance of any license notices (including + copyright notices, patent notices, disclaimers of warranty, or limitations + of liability) contained within the Source Code Form of the Covered + Software, except that You may alter any license notices to the extent + required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + + You may choose to offer, and to charge a fee for, warranty, support, + indemnity or liability obligations to one or more recipients of Covered + Software. However, You may do so only on Your own behalf, and not on behalf + of any Contributor. You must make it absolutely clear that any such + warranty, support, indemnity, or liability obligation is offered by You + alone, and You hereby agree to indemnify every Contributor for any + liability incurred by such Contributor as a result of warranty, support, + indemnity or liability terms You offer. You may include additional + disclaimers of warranty and limitations of liability specific to any + jurisdiction. + +4. Inability to Comply Due to Statute or Regulation + + If it is impossible for You to comply with any of the terms of this License + with respect to some or all of the Covered Software due to statute, judicial + order, or regulation then You must: (a) comply with the terms of this License + to the maximum extent possible; and (b) describe the limitations and the code + they affect. Such description must be placed in a text file included with all + distributions of the Covered Software under this License. Except to the + extent prohibited by statute or regulation, such description must be + sufficiently detailed for a recipient of ordinary skill to be able to + understand it. + +5. Termination + +5.1. The rights granted under this License will terminate automatically if You + fail to comply with any of its terms. However, if You become compliant, + then the rights granted under this License from a particular Contributor + are reinstated (a) provisionally, unless and until such Contributor + explicitly and finally terminates Your grants, and (b) on an ongoing basis, + if such Contributor fails to notify You of the non-compliance by some + reasonable means prior to 60 days after You have come back into compliance. + Moreover, Your grants from a particular Contributor are reinstated on an + ongoing basis if such Contributor notifies You of the non-compliance by + some reasonable means, this is the first time You have received notice of + non-compliance with this License from such Contributor, and You become + compliant prior to 30 days after Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent + infringement claim (excluding declaratory judgment actions, counter-claims, + and cross-claims) alleging that a Contributor Version directly or + indirectly infringes any patent, then the rights granted to You by any and + all Contributors for the Covered Software under Section 2.1 of this License + shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user + license agreements (excluding distributors and resellers) which have been + validly granted by You or Your distributors under this License prior to + termination shall survive termination. + +6. Disclaimer of Warranty + + Covered Software is provided under this License on an “as is” basis, without + warranty of any kind, either expressed, implied, or statutory, including, + without limitation, warranties that the Covered Software is free of defects, + merchantable, fit for a particular purpose or non-infringing. The entire + risk as to the quality and performance of the Covered Software is with You. + Should any Covered Software prove defective in any respect, You (not any + Contributor) assume the cost of any necessary servicing, repair, or + correction. This disclaimer of warranty constitutes an essential part of this + License. No use of any Covered Software is authorized under this License + except under this disclaimer. + +7. Limitation of Liability + + Under no circumstances and under no legal theory, whether tort (including + negligence), contract, or otherwise, shall any Contributor, or anyone who + distributes Covered Software as permitted above, be liable to You for any + direct, indirect, special, incidental, or consequential damages of any + character including, without limitation, damages for lost profits, loss of + goodwill, work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses, even if such party shall have been + informed of the possibility of such damages. This limitation of liability + shall not apply to liability for death or personal injury resulting from such + party’s negligence to the extent applicable law prohibits such limitation. + Some jurisdictions do not allow the exclusion or limitation of incidental or + consequential damages, so this exclusion and limitation may not apply to You. + +8. Litigation + + Any litigation relating to this License may be brought only in the courts of + a jurisdiction where the defendant maintains its principal place of business + and such litigation shall be governed by laws of that jurisdiction, without + reference to its conflict-of-law provisions. Nothing in this Section shall + prevent a party’s ability to bring cross-claims or counter-claims. + +9. Miscellaneous + + This License represents the complete agreement concerning the subject matter + hereof. If any provision of this License is held to be unenforceable, such + provision shall be reformed only to the extent necessary to make it + enforceable. Any law or regulation which provides that the language of a + contract shall be construed against the drafter shall not be used to construe + this License against a Contributor. + + +10. Versions of the License + +10.1. New Versions + + Mozilla Foundation is the license steward. Except as provided in Section + 10.3, no one other than the license steward has the right to modify or + publish new versions of this License. Each version will be given a + distinguishing version number. + +10.2. Effect of New Versions + + You may distribute the Covered Software under the terms of the version of + the License under which You originally received the Covered Software, or + under the terms of any subsequent version published by the license + steward. + +10.3. Modified Versions + + If you create software not governed by this License, and you want to + create a new license for such software, you may create and use a modified + version of this License if you rename the license and remove any + references to the name of the license steward (except to note that such + modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses + If You choose to distribute Source Code Form that is Incompatible With + Secondary Licenses under the terms of this version of the License, the + notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice + + This Source Code Form is subject to the + terms of the Mozilla Public License, v. + 2.0. If a copy of the MPL was not + distributed with this file, You can + obtain one at + http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular file, then +You may include the notice in a location (such as a LICENSE file in a relevant +directory) where a recipient would be likely to look for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - “Incompatible With Secondary Licenses” Notice + + This Source Code Form is “Incompatible + With Secondary Licenses”, as defined by + the Mozilla Public License, v. 2.0. + +================================================================ + github.com/hashicorp/hcl https://github.com/hashicorp/hcl ---------------------------------------------------------------- diff --git a/build/stdbuild.yml b/build/stdbuild.yml index f6f9d67..ef23c5d 100644 --- a/build/stdbuild.yml +++ b/build/stdbuild.yml @@ -99,11 +99,13 @@ stages: - name: go list modules type: run command: 'go list -json -m all' - output: '{{ .artifactsDir }}/list.tmp' + output: + variable: nancy - name: nancy type: run command: 'nancy sleuth --quiet' - input: '{{ .artifactsDir }}/list.tmp' + input: + variable: nancy logStdOut: true - name: coverage @@ -124,7 +126,8 @@ stages: - name: credits type: run command: 'gocredits' - output: 'CREDITS' + output: + path: 'CREDITS' # release releases the project using goreleaser - name: snapshot_release @@ -135,20 +138,22 @@ stages: value: '{{ .resDir }}//Dockerfile.release' - name: releaseHeader optional: true - skipTemplate: true - file: '{{ .resDir }}/header.tplt' + skipExpand: true + path: '{{ .resDir }}/header.tplt' - name: releaseFooter optional: true - skipTemplate: true - file: '{{ .resDir }}/footer.tplt' + skipExpand: true + path: '{{ .resDir }}/footer.tplt' tasks: - name: prepare release script type: template delims: left: '[[' right: ']]' - template: '{{ .resDir }}/goreleaser.ytpl' - output: '{{ .goreleaserCfg }}' + template: + path: '{{ .resDir }}/goreleaser.ytpl' + output: + path: '{{ .goreleaserCfg }}' - name: go releaser type: run @@ -169,11 +174,11 @@ stages: value: '{{ .resDir }}/Dockerfile.release' - name: releaseHeader optional: true - skipTemplate: true + skipExpand: true file: '{{ .resDir }}/header.tplt' - name: releaseFooter optional: true - skipTemplate: true + skipExpand: true file: '{{ .resDir }}/footer.tplt' tasks: @@ -182,8 +187,10 @@ stages: delims: left: '[[' right: ']]' - template: '{{ .resDir }}/goreleaser.ytpl' - output: '{{ .goreleaserCfg }}' + template: + path: '{{ .resDir }}/goreleaser.ytpl' + output: + path: '{{ .goreleaserCfg }}' - name: go releaser type: run diff --git a/go.mod b/go.mod index 90c95d6..a2d17d2 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.16 require ( github.com/apex/log v1.9.0 + github.com/hashicorp/go-multierror v1.1.1 github.com/mitchellh/mapstructure v1.4.1 github.com/pkg/errors v0.8.1 github.com/spf13/cobra v1.2.1 diff --git a/go.sum b/go.sum index 37431b4..9c4b796 100644 --- a/go.sum +++ b/go.sum @@ -152,11 +152,14 @@ github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORR github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= diff --git a/internal/cmd/initcoonfig.yml b/internal/cmd/initcoonfig.yml index f177a54..18d387d 100644 --- a/internal/cmd/initcoonfig.yml +++ b/internal/cmd/initcoonfig.yml @@ -10,7 +10,7 @@ name: "init sample mission" version: '1.0' # sequences can be used to specify what stages are executed. If they are not included all stages are run in file order. -# when oone or more sequences are defined the stages are run in the order of the requested sequence(s). +# when one or more sequences are defined the stages are run in the order of the requested sequence(s). # in the cas below the command would be: cirocket launch run # sequences: # run: @@ -19,13 +19,17 @@ version: '1.0' # params are configuration settings. The are accessible in all settings that support template expansion # params can be specified as part of the mission, stage or task. Params are inherited and can be overbidden # all params must have aa name, the key by which they are accessible name: fred is accessible as {{ .fred }} in templates. -# param values can be specified using the value tag or read from a file. Both are subject to template expansion. +# param values can be specified using the value tag or read from a file or web url. Both are subject to template expansion. # If both value and file are specified the final value is the concatenation of the value property and file contents. +# skipExpand stops template expansion occurring. If optional is true no error will be raised if the file or url is not found params: - name: welcome value: hello world #- name: secret - # file: somefile.txt + # path: some_file.txt + # url: "https//..." + # skipExpand: false + # optional: false # env contains environment variables that are defined for use in template expansion or passed to any sub processes executed # env variables are subject to param expansion, so can for example be passed secrets from params. @@ -85,35 +89,60 @@ stages: command: go args: - version - #glob: true - #logStdOut: true + # glob: false + # logStdOut: false # sub process output is either sent to a file, the log or to the host applications stdout. - # output allows a file to be specified, appendOutput controls if the file should be appended to or truncated + # redirection uses the input, output and error sub keys + # output allows a file or variable to be specified, append controls if the file should be appended to or truncated # logStdOut sends log output, if not going to a file to the log, otherwise its send to stdout. - # output: filename - # appendOutput: true + # output: + # path: filename + # variable: exported_variable + # append: false # logStdOut: true - # a input file can be specified in place of stdin - # input: input - - # error output normally goes to the log - # it can be directed to a file, merged with the output file or direct to stderr - # error: error_file - # appendError: true - # directStdErr: true + # a input file can be specified in place of stdin, sources include variables, files, urls or inline + # all arguments support template expansion. This can be disabled with the skipExpand arg. The timeout setting + # allows a request timeout limit to be specified. If blank default is 30 seconds. + # input: + # inline: test + # path: file + # url: url + # variable: exported_variable + + # error output normally goes to the log, this can be modified using the error sub key + # it can be directed to a file, merged with the output file or direct to the host processes stderr. + # error: + # path: filename + # variable: exported_variable + # append: false + # directStdErr: false # template tasks are used to run a go template - name: template_example type: template + # template input is defined by the template sub key and its output by the output key + # inline templates are not expanded prior to template processing + # template: + # inline: test + # path: file + # url: url + # variable: exported_variable + + # output: + # path: filename + # variable: exported_variable + # append: false + # either a template or inline property must be specified. # template is the name of the go template file # inline is an inline expansion #template: file - inline: | - Say {{.welcome}}! + template: + inline: | + Say {{.welcome}}! # template output is either sent to a file or to the host applications stdout. # output: filename @@ -128,12 +157,18 @@ stages: # - 'file-1' - # fetch pulls files from a request urls + # fetch pulls data from files, inline statements and urls into exported variables or local files. - name: fetch_task type: fetch # resources: - # - url: 'https://raw.githubusercontent.com/nehemming/cirocket/master/README.md' - # output: testdata/readme.tmp - + # - source: + # inline: test + # path: file + # url: url + # variable: exported_variable + # output: + # path: 'testdata/readme.tmp' + # variable: exported_variable + # append: false # end of file \ No newline at end of file diff --git a/internal/cmd/launch.go b/internal/cmd/launch.go index d8622cd..63e6241 100644 --- a/internal/cmd/launch.go +++ b/internal/cmd/launch.go @@ -1,20 +1,58 @@ package cmd import ( + "fmt" + "strings" + "github.com/nehemming/cirocket/pkg/rocket" "github.com/spf13/cobra" "github.com/spf13/viper" ) +func getCliParams(cmd *cobra.Command) ([]rocket.Param, error) { + valueParams, err := cmd.Flags().GetStringArray(flagParams) + if err != nil { + return nil, err + } + return parseParams(valueParams) +} + +func parseParams(valueParams []string) ([]rocket.Param, error) { + params := make([]rocket.Param, len(valueParams)) + + for i, nv := range valueParams { + slice := strings.SplitN(nv, "=", 2) + + if len(slice) != 2 { + return nil, fmt.Errorf("param[%d] %s is not formed as name=value", i, nv) + } + + params[i] = rocket.Param{Name: slice[0], Value: slice[1]} + } + + return params, nil +} + func (cli *cli) runFireCmd(cmd *cobra.Command, args []string) error { // Check that the init process found a config file if cli.initError != nil { return cli.initError } + // Handle params + params, err := getCliParams(cmd) + if err != nil { + return err + } + // Attempt to launch mission - return rocket.Default().LaunchMission(cli.ctx, viper.ConfigFileUsed(), viper.AllSettings(), args...) + return rocket.Default(). + LaunchMissionWithParams(cli.ctx, viper.ConfigFileUsed(), + viper.AllSettings(), params, args...) } +const flagParams = "params" + func (cli *cli) bindLaunchFlagsAndConfig(cmd *cobra.Command) { + cmd.Flags().StringArray(flagParams, nil, "supply parameter values to the mission, multiple params flags can be provided") } diff --git a/internal/cmd/launch_test.go b/internal/cmd/launch_test.go new file mode 100644 index 0000000..4e9118b --- /dev/null +++ b/internal/cmd/launch_test.go @@ -0,0 +1,53 @@ +package cmd + +import "testing" + +func TestParseParamsEmpty(t *testing.T) { + var list []string + + r, err := parseParams(list) + if err != nil || len(r) > 0 { + t.Error("unexpected", err, len(r)) + } +} + +func TestParseParamsSingle(t *testing.T) { + list := []string{"abc=123"} + + r, err := parseParams(list) + if err != nil || len(r) != 1 { + t.Error("unexpected", err, len(r)) + return + } + + if r[0].Name != "abc" { + t.Error("unexpected name", r[0].Name) + } + if r[0].Value != "123" { + t.Error("unexpected value", r[0].Name, r[0].Value) + } +} + +func TestParseParamsMultiple(t *testing.T) { + list := []string{"abc=123", "def=456,7=8"} + + r, err := parseParams(list) + if err != nil || len(r) != 2 { + t.Error("unexpected", err, len(r)) + return + } + + if r[0].Name != "abc" { + t.Error("unexpected name", r[0].Name) + } + if r[0].Value != "123" { + t.Error("unexpected value", r[0].Name, r[0].Value) + } + + if r[1].Name != "def" { + t.Error("unexpected name", r[1].Name) + } + if r[1].Value != "456,7=8" { + t.Error("unexpected value", r[1].Name, r[0].Value) + } +} diff --git a/pkg/loggee/log.go b/pkg/loggee/log.go index 66f595f..af14670 100644 --- a/pkg/loggee/log.go +++ b/pkg/loggee/log.go @@ -49,6 +49,11 @@ func SetLogger(logger Logger) { mustHaveLogger() } +func Default() Logger { + mustHaveLogger() + return defaultLog +} + // mustHaveLogger checks that a logger has been set. If there is no logger this method will panic. func mustHaveLogger() { if defaultLog == nil { diff --git a/pkg/providers/closerlist.go b/pkg/providers/closerlist.go new file mode 100644 index 0000000..5ed17e4 --- /dev/null +++ b/pkg/providers/closerlist.go @@ -0,0 +1,47 @@ +package providers + +import ( + "io" + + "github.com/hashicorp/go-multierror" + "github.com/pkg/errors" +) + +// CloserList is a list of io.Closers. +type CloserList struct { + closers []io.Closer +} + +// NewCloserList creates a new closer list. +func NewCloserList(items ...interface{}) *CloserList { + return new(CloserList).Append(items...) +} + +// Append a closer or slice of closers to the closerlist. +func (cl *CloserList) Append(items ...interface{}) *CloserList { + for _, item := range items { + switch v := item.(type) { + case io.Closer: + cl.closers = append(cl.closers, v) + case []io.Closer: + cl.closers = append(cl.closers, v...) + } + } + + return cl +} + +// Close closes all closers in the list returning all errors as a single error. +func (cl *CloserList) Close() error { + var err error + + for i, c := range cl.closers { + e := c.Close() + + if e != nil { + err = multierror.Append(err, errors.Wrapf(e, "close[%d]", i)) + } + } + + return bindMultiErrorFormatting(err) +} diff --git a/pkg/providers/closerlist_test.go b/pkg/providers/closerlist_test.go new file mode 100644 index 0000000..5feeeec --- /dev/null +++ b/pkg/providers/closerlist_test.go @@ -0,0 +1,253 @@ +package providers + +import ( + "fmt" + "io" + "testing" +) + +type testCloser struct { + counter int +} + +func (tc *testCloser) Close() error { + tc.counter++ + + return nil +} + +type testErrorCloser struct { + id int +} + +func (tc *testErrorCloser) Close() error { + return fmt.Errorf("err %d", tc.id) +} + +func TestCloserListNewEmpty(t *testing.T) { + cl := NewCloserList() + + if len(cl.closers) != 0 { + t.Error("NewCloserList is not empty", len(cl.closers)) + } +} + +func TestCloserListNewSingleCloser(t *testing.T) { + c := &testCloser{} + + cl := NewCloserList(c) + + if len(cl.closers) != 1 { + t.Error("NewCloserList is not 1", len(cl.closers)) + } + + cl.Close() + + if c.counter != 1 { + t.Error("CloserList did not close") + } +} + +func TestCloserListNewTwoCloser(t *testing.T) { + c := &testCloser{} + c1 := &testCloser{} + + cl := NewCloserList(c, c1) + + if len(cl.closers) != 2 { + t.Error("NewCloserList is not 2", len(cl.closers)) + } + + cl.Close() + + if c.counter != 1 { + t.Error("CloserList did not close c") + } + + if c1.counter != 1 { + t.Error("CloserList did not close c1") + } +} + +func TestCloserListNewMultipleClosers(t *testing.T) { + c := make([]io.Closer, 5) + for i := 0; i < len(c); i++ { + c[i] = &testCloser{} + } + + cl := NewCloserList(c) + + if len(cl.closers) != len(c) { + t.Error("NewCloserList is not", len(c), len(cl.closers)) + } + + cl.Close() + + for i := 0; i < len(c); i++ { + if c[i].(*testCloser).counter != 1 { + t.Error("CloserList did not close", i) + } + } +} + +func TestCloserListNewMultipleDeepClosers(t *testing.T) { + c := make([]io.Closer, 5) + for i := 0; i < len(c); i++ { + c[i] = &testCloser{} + } + + cl := NewCloserList(c) + cSingle := &testCloser{} + + cl2 := NewCloserList(cl, cSingle) + + if len(cl2.closers) != 2 { + t.Error("NewCloserList is shallow", len(cl.closers)) + } + + cl2.Close() + + for i := 0; i < len(c); i++ { + if c[i].(*testCloser).counter != 1 { + t.Error("CloserList did not close", i) + } + } + + if cSingle.counter != 1 { + t.Error("CloserList did not close cSingle") + } +} + +func TestCloserListAppendSingleCloser(t *testing.T) { + cl := NewCloserList() + + c := &testCloser{} + + cl.Append(c) + + if len(cl.closers) != 1 { + t.Error("NewCloserList is not 1", len(cl.closers)) + } + + cl.Close() + + if c.counter != 1 { + t.Error("CloserList did not close") + } +} + +func TestCloserListAppendTwoCloser(t *testing.T) { + cl := NewCloserList() + + c := &testCloser{} + c1 := &testCloser{} + + cl.Append(c, c1) + + if len(cl.closers) != 2 { + t.Error("NewCloserList is not 2", len(cl.closers)) + } + + err := cl.Close() + if err != nil { + t.Error("close", err) + } + + if c.counter != 1 { + t.Error("CloserList did not close c") + } + + if c1.counter != 1 { + t.Error("CloserList did not close c1") + } +} + +func TestCloserListAppendMultipleClosers(t *testing.T) { + cl := NewCloserList() + + c := make([]io.Closer, 5) + for i := 0; i < len(c); i++ { + c[i] = &testCloser{} + } + + cl.Append(c) + + if len(cl.closers) != len(c) { + t.Error("NewCloserList is not", len(c), len(cl.closers)) + } + + err := cl.Close() + if err != nil { + t.Error("close", err) + } + + for i := 0; i < len(c); i++ { + if c[i].(*testCloser).counter != 1 { + t.Error("CloserList did not close", i) + } + } +} + +func TestCloserLisAppendMultipleDeepClosers(t *testing.T) { + cl := NewCloserList() + + c := make([]io.Closer, 5) + for i := 0; i < len(c); i++ { + c[i] = &testCloser{} + } + + cl.Append(c) + cSingle := &testCloser{} + + cl2 := NewCloserList() + + cl2.Append(cl, cSingle) + + if len(cl2.closers) != 2 { + t.Error("NewCloserList is shallow", len(cl.closers)) + } + + err := cl2.Close() + if err != nil { + t.Error("close", err) + } + for i := 0; i < len(c); i++ { + if c[i].(*testCloser).counter != 1 { + t.Error("CloserList did not close", i) + } + } + + if cSingle.counter != 1 { + t.Error("CloserList did not close cSingle") + } +} + +func TestCloserListNewSingleCloserErrors(t *testing.T) { + c := &testErrorCloser{} + + cl := NewCloserList(c) + + if len(cl.closers) != 1 { + t.Error("NewCloserList is not 1", len(cl.closers)) + } + + err := cl.Close() + if err == nil { + t.Error("No error on close") + } +} + +func TestCloserListNewMultipleCloserErrors(t *testing.T) { + c := &testErrorCloser{} + + cl := NewCloserList(c, &testErrorCloser{1}) + + if len(cl.closers) != 2 { + t.Error("NewCloserList is not 2", len(cl.closers)) + } + + err := cl.Close() + if err == nil { + t.Error("No error on close") + } +} diff --git a/pkg/providers/fileprovider.go b/pkg/providers/fileprovider.go new file mode 100644 index 0000000..a564555 --- /dev/null +++ b/pkg/providers/fileprovider.go @@ -0,0 +1,156 @@ +package providers + +import ( + "bytes" + "context" + "io" + "os" + + "github.com/pkg/errors" +) + +type ( + FileDetail interface { + FilePath() string + IOMode() IOMode + FileMode() os.FileMode + InMode(mode IOMode) bool + } + + fileResourceProvider struct { + filePath string + ioMode IOMode + fileMode os.FileMode + optional bool + } + + nopReaderCloser struct { + reader io.Reader + } + + nopWriterCloser struct { + writer io.Writer + } +) + +// NewNonClosingReaderProvider attaches an existing reader (i.e. stdin) to a provider. +func NewNonClosingReaderProvider(reader io.Reader) ResourceProvider { + return &nopReaderCloser{ + reader: reader, + } +} + +// NewNonClosingWriterProvider attaches an existing writer (i.e. stdout) to a provider. +func NewNonClosingWriterProvider(writer io.Writer) ResourceProvider { + return &nopWriterCloser{ + writer: writer, + } +} + +// NewFileProvider creates a file provider. +func NewFileProvider(path string, ioMode IOMode, fileMode os.FileMode, optional bool) (ResourceProvider, error) { + if path == "" { + return nil, errors.New("path is blank") + } + + if ioMode&(IOModeOutput|IOModeError) != IOModeNone { + if ioMode&(IOModeTruncate|IOModeAppend) == (IOModeTruncate | IOModeAppend) { + return nil, errors.New("both truncate and append have been specified, please select only one") + } else if ioMode&(IOModeTruncate|IOModeAppend) == IOModeNone { + return nil, errors.New("neither truncate nor append have been specified, please select only one") + } + // Supports output + } else if ioMode&IOModeInput != IOModeInput { + // not in or out + return nil, errors.New("mode is neither input nor output") + } + + return &fileResourceProvider{ + filePath: path, + ioMode: ioMode, + fileMode: fileMode, + optional: optional, + }, nil +} + +func (rc *nopReaderCloser) OpenWrite(ctx context.Context) (io.WriteCloser, error) { + return nil, errors.New("output is not supported") +} + +func (rc *nopReaderCloser) OpenRead(ctx context.Context) (io.ReadCloser, error) { + return rc, nil +} + +func (rc *nopReaderCloser) Read(p []byte) (n int, err error) { + return rc.reader.Read(p) +} + +func (rc *nopReaderCloser) Close() error { + return nil +} + +func (wc *nopWriterCloser) OpenWrite(ctx context.Context) (io.WriteCloser, error) { + return wc, nil +} + +func (wc *nopWriterCloser) OpenRead(ctx context.Context) (io.ReadCloser, error) { + return nil, errors.New("input is not supported") +} + +func (wc *nopWriterCloser) Write(p []byte) (n int, err error) { + return wc.writer.Write(p) +} + +func (wc *nopWriterCloser) Close() error { + return nil +} + +func (fp *fileResourceProvider) FilePath() string { + return fp.filePath +} + +func (fp *fileResourceProvider) IOMode() IOMode { + return fp.ioMode +} + +func (fp *fileResourceProvider) FileMode() os.FileMode { + return fp.fileMode +} + +func (fp *fileResourceProvider) InMode(mode IOMode) bool { + return fp.ioMode&mode == mode +} + +func (fp *fileResourceProvider) OpenRead(ctx context.Context) (io.ReadCloser, error) { + if fp.ioMode&IOModeInput != IOModeInput { + return nil, errors.New("input is not supported") + } + + rc, err := os.Open(fp.filePath) + if err != nil { + if !os.IsNotExist(err) || !fp.optional { + return nil, err + } + + // Return an empty reader + return &nopReaderCloser{ + reader: bytes.NewBufferString(""), + }, nil + } + + return rc, err +} + +func (fp *fileResourceProvider) OpenWrite(ctx context.Context) (io.WriteCloser, error) { + if fp.ioMode&(IOModeOutput|IOModeError) == IOModeNone { + return nil, errors.New("output is not supported") + } + + if (fp.ioMode & IOModeTruncate) == IOModeTruncate { + return os.OpenFile(fp.filePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, fp.fileMode) + } else if (fp.ioMode & IOModeAppend) == IOModeAppend { + return os.OpenFile(fp.filePath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, fp.fileMode) + } else { + panic("validation should have caught missing append or truncate") + } +} diff --git a/pkg/providers/fileprovider_test.go b/pkg/providers/fileprovider_test.go new file mode 100644 index 0000000..b658a65 --- /dev/null +++ b/pkg/providers/fileprovider_test.go @@ -0,0 +1,313 @@ +package providers + +import ( + "bytes" + "context" + "io" + "os" + "strings" + "testing" +) + +func TestNewNonClosingReaderProvider(t *testing.T) { + buf := bytes.NewBufferString("hello") + + provider := NewNonClosingReaderProvider(buf) + if provider == nil { + t.Error("unexpected nil provider") + } + + w, err := provider.OpenWrite(context.Background()) + if err == nil || w != nil { + t.Error("error expected writer", err, w) + } + + r, err := provider.OpenRead(context.Background()) + if err != nil || r == nil { + t.Error("error unexpected reader", err, r) + } + r.Close() + + b, err := io.ReadAll(r) + if err != nil || string(b) != "hello" { + t.Error("error unexpected read", err, string(b)) + } +} + +func TestNewNonClosingWriterProvider(t *testing.T) { + buf := new(bytes.Buffer) + + provider := NewNonClosingWriterProvider(buf) + if provider == nil { + t.Error("unexpected nil provider") + } + + r, err := provider.OpenRead(context.Background()) + if err == nil || r != nil { + t.Error("error expected reader", err, r) + } + + w, err := provider.OpenWrite(context.Background()) + if err != nil || w == nil { + t.Error("error unexpected writer", err, r) + } + + w.Close() + + n, err := w.Write([]byte("hello")) + if err != nil || n != 5 { + t.Error("error unexpected write", err, n) + } + + s := buf.String() + if s != "hello" { + t.Error("error unexpected read", err, s) + } +} + +func TestNewNewFileProviderWrongType(t *testing.T) { + _, err := NewFileProvider("fileprovider.go", IOModeOutput, 0, false) + if err == nil { + t.Error("expected error", err) + } + if err.Error() != "neither truncate nor append have been specified, please select only one" { + t.Error("error mismatch", err) + } +} + +func TestNewNewFileProviderWrongTypeError(t *testing.T) { + _, err := NewFileProvider("fileprovider.go", IOModeError, 0, false) + if err == nil { + t.Error("expected error", err) + } + if err.Error() != "neither truncate nor append have been specified, please select only one" { + t.Error("error mismatch", err) + } +} + +func TestNewNewFileProviderModesError(t *testing.T) { + _, err := NewFileProvider("fileprovider.go", IOModeError|IOModeAppend|IOModeTruncate, 0, false) + if err == nil { + t.Error("expected error", err) + } + if err.Error() != "both truncate and append have been specified, please select only one" { + t.Error("error mismatch", err) + } +} + +func TestNewNewFileProviderNoIOModeError(t *testing.T) { + _, err := NewFileProvider("fileprovider.go", IOModeAppend|IOModeTruncate, 0, false) + if err == nil { + t.Error("expected error", err) + } + if err.Error() != "mode is neither input nor output" { + t.Error("error mismatch", err) + } +} + +func TestNewNewFileProviderBlankPath(t *testing.T) { + _, err := NewFileProvider("", IOModeAppend|IOModeTruncate, 0, false) + if err == nil { + t.Error("expected error", err) + } + if err.Error() != "path is blank" { + t.Error("error mismatch", err) + } +} + +func TestNewFileProviderOpensForRead(t *testing.T) { + rp, err := NewFileProvider("fileprovider.go", IOModeInput, 0, false) + if err != nil { + t.Error("unexpected error", err) + } + + reader, err := rp.OpenRead(context.Background()) + if err != nil { + t.Error("unexpected error", err) + } + + defer reader.Close() + + b, err := io.ReadAll(reader) + if err != nil { + t.Error("unexpected error", err) + } + + if !strings.HasPrefix(string(b), "package providers") { + t.Error("unexpected error", err, string(b)) + } +} + +func TestNewFileProviderOpensForReaderrorNotFound(t *testing.T) { + rp, err := NewFileProvider("no_fileprovider.go", IOModeInput, 0, false) + if err != nil { + t.Error("unexpected error", err) + } + + _, err = rp.OpenRead(context.Background()) + if err == nil { + t.Error("expected error") + } +} + +func TestNewFileProviderOpensForReadOptional(t *testing.T) { + rp, err := NewFileProvider("no_fileprovider.go", IOModeInput, 0, true) + if err != nil { + t.Error("unexpected error", err) + } + + reader, err := rp.OpenRead(context.Background()) + if err != nil { + t.Error("unexpected error", err) + } + + defer reader.Close() + + b, err := io.ReadAll(reader) + if err != nil { + t.Error("unexpected error", err) + } + + if len(b) != 0 { + t.Error("non zero missing optional ", len(b)) + } +} + +func TestNewNewFileProviderErrorsForWrite(t *testing.T) { + rp, err := NewFileProvider("fileprovider.go", IOModeInput, 0, false) + if err != nil { + t.Error("unexpected error", err) + } + + _, err = rp.OpenWrite(context.Background()) + if err == nil { + t.Error("expected error", err) + } + + if err.Error() != "output is not supported" { + t.Error("expected error", err) + } +} + +func TestNewNewFileProviderErrorsForRead(t *testing.T) { + err := os.MkdirAll("testdata", 0777) + if err != nil { + panic(err) + } + + rp, err := NewFileProvider("testdata/new.dat", IOModeError|IOModeTruncate, 0, false) + if err != nil { + t.Error("unexpected error", err) + } + + _, err = rp.OpenRead(context.Background()) + if err == nil { + t.Error("expected error", err) + } + + if err.Error() != "input is not supported" { + t.Error("expected error", err) + } +} + +func TestNewNewFileProviderTruncatesFile(t *testing.T) { + err := os.MkdirAll("testdata", 0777) + if err != nil { + panic(err) + } + + err = os.WriteFile("testdata/trunc.dat", []byte("bad"), 0666) + if err != nil { + panic(err) + } + + rp, err := NewFileProvider("testdata/trunc.dat", IOModeOutput|IOModeTruncate, 0, false) + if err != nil { + t.Error("unexpected error", err) + } + + w, err := rp.OpenWrite(context.Background()) + if err != nil { + t.Error("expected error open", err) + } + + _, err = w.Write([]byte("good")) + if err != nil { + t.Error("expected error write", err) + } + + b, err := os.ReadFile("testdata/trunc.dat") + if err != nil { + panic(err) + } + + if string(b) != "good" { + t.Error("expected write data", string(b)) + } +} + +func TestNewNewFileProviderAppendFile(t *testing.T) { + err := os.MkdirAll("testdata", 0777) + if err != nil { + panic(err) + } + + err = os.WriteFile("testdata/append.dat", []byte("hello "), 0666) + if err != nil { + panic(err) + } + + rp, err := NewFileProvider("testdata/append.dat", IOModeOutput|IOModeAppend, 0, false) + if err != nil { + t.Error("unexpected error", err) + } + + w, err := rp.OpenWrite(context.Background()) + if err != nil { + t.Error("expected error open", err) + } + + _, err = w.Write([]byte("good")) + if err != nil { + t.Error("expected error write", err) + } + + b, err := os.ReadFile("testdata/append.dat") + if err != nil { + panic(err) + } + + if string(b) != "hello good" { + t.Error("expected write data", string(b)) + } + + err = os.WriteFile("testdata/append.dat", []byte("hello "), 0666) + if err != nil { + panic(err) + } +} + +func TestNewNewFileProviderFileDetails(t *testing.T) { + res, err := NewFileProvider("fileprovider.go", IOModeInput, 0666, false) + if err != nil { + t.Error("unexpected error", err) + } + + fp, ok := res.(FileDetail) + if !ok { + t.Error("error FileDetail not implemented") + return + } + + if fp.FilePath() != "fileprovider.go" { + t.Error("file path", fp.FilePath()) + } + + if fp.IOMode() != IOModeInput { + t.Error("io mode", fp.IOMode()) + } + + if fp.FileMode() != 0666 { + t.Error("mode", fp.FileMode()) + } +} diff --git a/pkg/providers/iomodes.go b/pkg/providers/iomodes.go new file mode 100644 index 0000000..cce836f --- /dev/null +++ b/pkg/providers/iomodes.go @@ -0,0 +1,52 @@ +package providers + +const ( + // IOModeInput file can be used for input. + IOModeInput = IOMode(1 << iota) + + // IOModeOutput file can be used for output. + IOModeOutput + + // IOModeError file can be used for errors. + IOModeError + + // IOModeTruncate file should be truncated. + IOModeTruncate + + // IOModeAppend file should be appended to. + IOModeAppend + + // IOModeNone is the default empty mde. + IOModeNone = IOMode(0) +) + +var ioModeMap = map[IOMode]rune{ + IOModeInput: 'i', + IOModeOutput: 'o', + IOModeError: 'e', + IOModeAppend: 'a', + IOModeTruncate: 't', +} + +var ioAllModes = []IOMode{ + IOModeInput, + IOModeOutput, + IOModeError, + IOModeAppend, + IOModeTruncate, +} + +// String converts IOMode to a string representation. +func (mode IOMode) String() string { + runes := make([]rune, len(ioAllModes)) + + for i, v := range ioAllModes { + if mode&v == v { + runes[i] = ioModeMap[v] + } else { + runes[i] = '-' + } + } + + return string(runes) +} diff --git a/pkg/providers/iomodes_test.go b/pkg/providers/iomodes_test.go new file mode 100644 index 0000000..7beeb59 --- /dev/null +++ b/pkg/providers/iomodes_test.go @@ -0,0 +1,73 @@ +package providers + +import ( + "fmt" + "testing" +) + +func modeString(mode IOMode) string { + return fmt.Sprintf("%d:%06b", mode, mode) +} + +func TestIOModes(t *testing.T) { + r := modeString(IOModeNone) + if r != "0:000000" { + t.Error("IOModeNone", r) + } + + r = modeString(IOModeInput) + if r != "1:000001" { + t.Error("IOModeInput", r) + } + + r = modeString(IOModeOutput) + if r != "2:000010" { + t.Error("IOModeOutput", r) + } + + r = modeString(IOModeError) + if r != "4:000100" { + t.Error("IOModeError", r) + } + + r = modeString(IOModeTruncate) + if r != "8:001000" { + t.Error("IOModeCreate", r) + } + + r = modeString(IOModeAppend) + if r != "16:010000" { + t.Error("IOModeCreate", r) + } +} + +func TestIOModesString(t *testing.T) { + r := IOModeNone.String() + if r != "-----" { + t.Error("IOModeNone", r) + } + + r = IOModeInput.String() + if r != "i----" { + t.Error("IOModeInput", r) + } + + r = IOModeOutput.String() + if r != "-o---" { + t.Error("IOModeOutput", r) + } + r = IOModeError.String() + if r != "--e--" { + t.Error("IOModeError", r) + } + + r = IOModeAppend.String() + if r != "---a-" { + t.Error("IOModeAppend", r) + } + + r = IOModeTruncate.String() + if r != "----t" { + t.Error("IOModeTruncate", r) + } +} diff --git a/pkg/providers/lineprovider.go b/pkg/providers/lineprovider.go new file mode 100644 index 0000000..e97e7e4 --- /dev/null +++ b/pkg/providers/lineprovider.go @@ -0,0 +1,63 @@ +package providers + +import ( + "bufio" + "context" + "io" + + "github.com/pkg/errors" +) + +type ( + LineFunc func(string) + + streamer struct { + lineFunc LineFunc + writer *io.PipeWriter + done chan struct{} + } +) + +// NewLineProvider writes lines of text to a output function. +func NewLineProvider(fn func(string)) ResourceProvider { + return LineFunc(fn) +} + +func (lf LineFunc) OpenRead(ctx context.Context) (io.ReadCloser, error) { + return nil, errors.New("input is not supported") +} + +func (lf LineFunc) OpenWrite(ctx context.Context) (io.WriteCloser, error) { + reader, writer := io.Pipe() + + st := &streamer{ + lineFunc: lf, + writer: writer, + done: make(chan struct{}), + } + + // Start reader, runs until pipe is closed + go func() { + defer close(st.done) + + in := bufio.NewScanner(reader) + + for in.Scan() { + st.lineFunc(in.Text()) + } + }() + + return st, nil +} + +func (st *streamer) Write(p []byte) (n int, err error) { + return st.writer.Write(p) +} + +func (st *streamer) Close() error { + st.writer.Close() + + // Wait for close to complete + <-st.done + return nil +} diff --git a/pkg/providers/lineprovider_test.go b/pkg/providers/lineprovider_test.go new file mode 100644 index 0000000..9399576 --- /dev/null +++ b/pkg/providers/lineprovider_test.go @@ -0,0 +1,45 @@ +package providers + +import ( + "context" + "testing" +) + +func TestNewLineProvider(t *testing.T) { + called := new(bool) + *called = false + + fn := func(s string) { + *called = true + if s != "hello" { + t.Error("expected hello", s) + } + } + + provider := NewLineProvider(fn) + + // Check read fails + _, err := provider.OpenRead(context.Background()) + if err == nil { + t.Error("error expected") + } + + writer, err := provider.OpenWrite(context.Background()) + if err != nil { + t.Error("error unexpected", err) + return + } + + _, err = writer.Write([]byte("hello")) + if err != nil { + t.Error("error unexpected", err) + writer.Close() + return + } + + writer.Close() + + if !*called { + t.Error("not called fn") + } +} diff --git a/pkg/providers/logprovider.go b/pkg/providers/logprovider.go new file mode 100644 index 0000000..e445c4f --- /dev/null +++ b/pkg/providers/logprovider.go @@ -0,0 +1,33 @@ +package providers + +import "github.com/nehemming/cirocket/pkg/loggee" + +const ( + // LogInfo is an informational level of logging. + LogInfo = Severity(iota) + // LogWarn is a warning level of logging. + LogWarn + // LogError is an error level of logging. + LogError +) + +// Severity is the level to log messages at in a log provider. +type Severity int + +// NewLogProvider creates provider that writes to a log. +func NewLogProvider(log loggee.Logger, severity Severity) ResourceProvider { + var logFunc LineFunc + + switch severity { + case LogInfo: + logFunc = log.Info + case LogWarn: + logFunc = log.Warn + case LogError: + logFunc = log.Error + default: + logFunc = func(_ string) {} + } + + return logFunc +} diff --git a/pkg/providers/logprovider_test.go b/pkg/providers/logprovider_test.go new file mode 100644 index 0000000..348a94b --- /dev/null +++ b/pkg/providers/logprovider_test.go @@ -0,0 +1,80 @@ +package providers + +import ( + "context" + "testing" + + "github.com/nehemming/cirocket/pkg/loggee/stdlog" +) + +func TestNewLogProviderSeverityError(t *testing.T) { + log := stdlog.New() + + lp := NewLogProvider(log, LogError) + + writer, err := lp.OpenWrite(context.Background()) + if err != nil { + t.Error("error unexpected", err) + return + } + defer writer.Close() + + _, err = writer.Write([]byte("hello")) + if err != nil { + t.Error("error unexpected", err) + } +} + +func TestNewLogProviderSeverityWarn(t *testing.T) { + log := stdlog.New() + + lp := NewLogProvider(log, LogWarn) + + writer, err := lp.OpenWrite(context.Background()) + if err != nil { + t.Error("error unexpected", err) + return + } + defer writer.Close() + + _, err = writer.Write([]byte("hello")) + if err != nil { + t.Error("error unexpected", err) + } +} + +func TestNewLogProviderSeverityInfo(t *testing.T) { + log := stdlog.New() + + lp := NewLogProvider(log, LogInfo) + + writer, err := lp.OpenWrite(context.Background()) + if err != nil { + t.Error("error unexpected", err) + return + } + defer writer.Close() + + _, err = writer.Write([]byte("hello")) + if err != nil { + t.Error("error unexpected", err) + } +} + +func TestNewLogProviderSeverityUndefined(t *testing.T) { + log := stdlog.New() + + lp := NewLogProvider(log, Severity(99)) + + writer, err := lp.OpenWrite(context.Background()) + if err != nil { + t.Error("error unexpected", err) + return + } + defer writer.Close() + + _, err = writer.Write([]byte("hello")) + if err != nil { + t.Error("error unexpected", err) + } +} diff --git a/pkg/providers/multipleerrors.go b/pkg/providers/multipleerrors.go new file mode 100644 index 0000000..509d2f4 --- /dev/null +++ b/pkg/providers/multipleerrors.go @@ -0,0 +1,33 @@ +package providers + +import ( + "fmt" + "strings" + + multierror "github.com/hashicorp/go-multierror" +) + +func multiErrorFormatter(es []error) string { + if len(es) == 1 { + return es[0].Error() + } + + text := make([]string, len(es)) + for i, err := range es { + text[i] = fmt.Sprintf("%s", err) + } + + return fmt.Sprintf( + "%d errors occurred: %s", + len(es), strings.Join(text, "; ")) +} + +func bindMultiErrorFormatting(err error) error { + if err != nil { + if multi, ok := err.(*multierror.Error); ok { + multi.ErrorFormat = multiErrorFormatter + } + } + + return err +} diff --git a/pkg/providers/multipleerrors_test.go b/pkg/providers/multipleerrors_test.go new file mode 100644 index 0000000..9e7f78e --- /dev/null +++ b/pkg/providers/multipleerrors_test.go @@ -0,0 +1,48 @@ +package providers + +import ( + "testing" + + multierror "github.com/hashicorp/go-multierror" + "github.com/pkg/errors" +) + +func TestMultipleErrorFormattingOnNil(t *testing.T) { + err := bindMultiErrorFormatting(nil) + if err != nil { + t.Error("Mismatch nil") + } +} + +func TestMultipleErrorFormattingSimple(t *testing.T) { + err := bindMultiErrorFormatting(errors.New("simple")) + + if err.Error() != "simple" { + t.Error("Mismatch simple", err) + } + + // err = multierror.Append(err, errors.Wrapf(e, "close[%d]", i)) +} + +func TestMultipleErrorFormattingSingle(t *testing.T) { + var err error + err = multierror.Append(err, errors.New("one")) + + err = bindMultiErrorFormatting(err) + + if err.Error() != "one" { + t.Error("Mismatch single", err) + } +} + +func TestMultipleErrorFormattingMultiple(t *testing.T) { + var err error + err = multierror.Append(err, errors.New("one")) + err = multierror.Append(err, errors.New("two")) + + err = bindMultiErrorFormatting(err) + + if err.Error() != "2 errors occurred: one; two" { + t.Error("Mismatch multiple", err) + } +} diff --git a/pkg/providers/providers.go b/pkg/providers/providers.go new file mode 100644 index 0000000..7e3eaaa --- /dev/null +++ b/pkg/providers/providers.go @@ -0,0 +1,60 @@ +package providers + +import ( + "context" + "io" + "sync" +) + +type ( + // IOMode indicates the modes of operation the file detail supports. + IOMode uint32 + + // ResourceID is a the ID of a resource. + ResourceID string + + // ResourceProvider creates resources. + // Implementers must implement both functions but can return an error + // for un supported requests. For example a Web Getter cannot be opened for writes so should return an error. + // All resources returned without an error should expect Close to be called on the ReadCloser or WriteCloser. + ResourceProvider interface { + OpenRead(ctx context.Context) (io.ReadCloser, error) + OpenWrite(ctx context.Context) (io.WriteCloser, error) + } + + // ResourceProviderMap is a map collection of resource prooviders. + ResourceProviderMap map[ResourceID]ResourceProvider + + idempotentCloser struct { + once sync.Once + closer io.Closer + } +) + +func (m ResourceProviderMap) Copy() ResourceProviderMap { + copy := make(ResourceProviderMap) + + for k, v := range m { + copy[k] = v + } + + return copy +} + +// NewIdempotentCloser creates a closer that calls the underlying closer once only. +func NewIdempotentCloser(closer io.Closer) io.Closer { + return &idempotentCloser{ + closer: closer, + } +} + +// Close the resource and return an error. +func (idem *idempotentCloser) Close() error { + var err error + + // calling close once, on that occasion err will be set, other calls err will + // remain nil as not set. + idem.once.Do(func() { err = idem.closer.Close() }) + + return err +} diff --git a/pkg/providers/providers_test.go b/pkg/providers/providers_test.go new file mode 100644 index 0000000..4ce4eaa --- /dev/null +++ b/pkg/providers/providers_test.go @@ -0,0 +1,21 @@ +package providers + +import "testing" + +func TestIdempotentCloserTest(t *testing.T) { + tc := &testCloser{} + + idc := NewIdempotentCloser(tc) + + idc.Close() + + if tc.counter != 1 { + t.Error("not closed") + } + + idc.Close() + + if tc.counter != 1 { + t.Error("over closed") + } +} diff --git a/pkg/providers/testdata/append.dat b/pkg/providers/testdata/append.dat new file mode 100644 index 0000000..dc106e6 --- /dev/null +++ b/pkg/providers/testdata/append.dat @@ -0,0 +1 @@ +hello \ No newline at end of file diff --git a/pkg/providers/testdata/trunc.dat b/pkg/providers/testdata/trunc.dat new file mode 100644 index 0000000..476e93d --- /dev/null +++ b/pkg/providers/testdata/trunc.dat @@ -0,0 +1 @@ +good \ No newline at end of file diff --git a/pkg/providers/urlprovider.go b/pkg/providers/urlprovider.go new file mode 100644 index 0000000..3d4e843 --- /dev/null +++ b/pkg/providers/urlprovider.go @@ -0,0 +1,75 @@ +package providers + +import ( + "bytes" + "context" + "fmt" + "io" + "net/http" + "time" + + "github.com/pkg/errors" + "golang.org/x/net/context/ctxhttp" +) + +type ( + urlResourceProvider struct { + url string + timeout time.Duration + optional bool + } +) + +// NewURLProvider creates a url provider. +func NewURLProvider(url string, timeout time.Duration, optional bool) (ResourceProvider, error) { + if url == "" { + return nil, errors.New("url is blank") + } + if timeout == 0 { + timeout = time.Second * 30 + } + + return &urlResourceProvider{ + url: url, + timeout: timeout, + optional: optional, + }, nil +} + +func (rp *urlResourceProvider) OpenWrite(ctx context.Context) (io.WriteCloser, error) { + return nil, errors.New("output is not supported") +} + +func (rp *urlResourceProvider) OpenRead(ctx context.Context) (io.ReadCloser, error) { + ctxTimeout, cancel := context.WithTimeout(ctx, rp.timeout) + defer cancel() + + req, err := http.NewRequest("GET", rp.url, nil) + if err != nil { + return nil, errors.Wrapf(err, "creating request for %s", rp.url) + } + + resp, err := ctxhttp.Do(ctxTimeout, nil, req) + if err != nil { + return nil, errors.Wrapf(err, "getting %s", rp.url) + } + defer resp.Body.Close() + + b := new(bytes.Buffer) + if !rp.optional || resp.StatusCode != http.StatusNotFound { + if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { + // Bad response + return nil, fmt.Errorf("response (%d) %s for %s", resp.StatusCode, resp.Status, rp.url) + } + + // make a copy so we can close the response body here, cannot escape the ctxTimeout context + _, err = io.Copy(b, resp.Body) + + if err != nil { + return nil, errors.Wrapf(err, "extracting body %s", rp.url) + } + } + + // Return the body + return &nopReaderCloser{b}, nil +} diff --git a/pkg/providers/urlprovider_test.go b/pkg/providers/urlprovider_test.go new file mode 100644 index 0000000..3dc78c7 --- /dev/null +++ b/pkg/providers/urlprovider_test.go @@ -0,0 +1,105 @@ +package providers + +import ( + "context" + "io" + "testing" + "time" +) + +func TestNewURLProviderBlankErrors(t *testing.T) { + p, err := NewURLProvider("", time.Second*5, false) + if err == nil || p != nil { + t.Error("error expected", err, p) + } +} + +func TestNewURLProvider(t *testing.T) { + p, err := NewURLProvider("somewhere", 0, false) + if err != nil || p == nil { + t.Error("error unexpected", err, p) + } + + up := p.(*urlResourceProvider) + if up.url != "somewhere" || up.timeout != time.Second*30 { + t.Error("values unexpected timeout default", up.url, up.timeout) + } + + p, err = NewURLProvider("somewhere", time.Second*5, false) + if err != nil || p == nil { + t.Error("error unexpected", err, p) + } + + up = p.(*urlResourceProvider) + + if up.url != "somewhere" || up.timeout != time.Second*5 { + t.Error("values unexpected", up.url, up.timeout) + } + + // Check write denied too + _, err = p.OpenWrite(context.Background()) + if err == nil { + t.Error("error expected for write open ", err) + } +} + +func TestNReadURLProvider(t *testing.T) { + p, err := NewURLProvider("https://raw.githubusercontent.com/nehemming/cirocket/master/README.md", time.Second*10, false) + if err != nil || p == nil { + t.Error("error unexpected", err, p) + } + ctx := context.Background() + + w, err := p.OpenRead(ctx) + if err != nil { + t.Error("open issue", err) + return + } + b, err := io.ReadAll(w) + if err != nil { + t.Error("read issue", err) + return + } + defer w.Close() + + if len(b) < 100 { + t.Error("reading b", len(b)) + } +} + +func TestNReadURLProviderOptional(t *testing.T) { + p, err := NewURLProvider("https://raw.githubusercontent.com/nehemming/cirocket/master/README-ntfound.md", time.Second*10, true) + if err != nil || p == nil { + t.Error("error unexpected", err, p) + } + ctx := context.Background() + + w, err := p.OpenRead(ctx) + if err != nil { + t.Error("open issue", err) + return + } + b, err := io.ReadAll(w) + if err != nil { + t.Error("read issue", err) + return + } + defer w.Close() + + if len(b) != 0 { + t.Error("reading b", len(b)) + } +} + +func TestNReadURLProviderErrorsNotFound(t *testing.T) { + p, err := NewURLProvider("https://raw.githubusercontent.com/nehemming/cirocket/master/README-ntfound.md", time.Second*10, false) + if err != nil || p == nil { + t.Error("error unexpected", err, p) + } + ctx := context.Background() + + _, err = p.OpenRead(ctx) + if err == nil { + t.Error("no open issue") + } +} diff --git a/pkg/rocket/builtin/fetch.go b/pkg/rocket/builtin/fetch.go index dcb844a..65b40ef 100644 --- a/pkg/rocket/builtin/fetch.go +++ b/pkg/rocket/builtin/fetch.go @@ -2,18 +2,13 @@ package builtin import ( "context" - "fmt" "io" - "net/http" - "os" "sync" - "time" "github.com/mitchellh/mapstructure" "github.com/nehemming/cirocket/pkg/loggee" "github.com/nehemming/cirocket/pkg/rocket" "github.com/pkg/errors" - "golang.org/x/net/context/ctxhttp" ) type ( @@ -21,20 +16,21 @@ type ( Fetch struct { Resources []FetchResource `mapstructure:"resources"` Log bool `mapstructure:"log"` - Timeout *int `mapstructure:"timeout"` } FetchResource struct { - URL string `mapstructure:"url"` - Output string `mapstructure:"output"` + Source rocket.InputSpec `mapstructure:"source"` + Output rocket.OutputSpec `mapstructure:"output"` } fetchType struct{} - urlFileTuple struct { - url string - out string + fetchOp struct { + source io.ReadCloser + target io.WriteCloser } + + fetchOps []*fetchOp ) func (fetchType) Type() string { @@ -48,17 +44,13 @@ func (fetchType) Prepare(ctx context.Context, capComm *rocket.CapComm, task rock return nil, errors.Wrap(err, "parsing template type") } - timeOut, err := getTimeOut(fetchCfg.Timeout) - if err != nil { - return nil, err - } - - resources, err := getResourcesList(ctx, capComm, fetchCfg.Resources) - if err != nil { - return nil, err - } + fn := func(execCtx context.Context) (err error) { + ops, err := getFetchOpsFromResourceList(ctx, capComm, fetchCfg.Resources) + if err != nil { + return + } + defer ops.Close() - fn := func(execCtx context.Context) error { // Create a context we can cancel fetchCtx, cancel := context.WithCancel(execCtx) defer cancel() @@ -75,7 +67,7 @@ func (fetchType) Prepare(ctx context.Context, capComm *rocket.CapComm, task rock // Consume errCh getting any errors form requests // Errors will cancel any remaining open requests - var err error + go func() { defer close(doneCh) for e := range errCh { @@ -93,20 +85,25 @@ func (fetchType) Prepare(ctx context.Context, capComm *rocket.CapComm, task rock }() // Run over requests - for _, res := range resources { + for i, res := range ops { // Check if exit requested - if ctx.Err() != nil { + if fetchCtx.Err() != nil { return nil } // Run in parallel wg.Add(1) - go func(working urlFileTuple) { + go func(index int, fetch *fetchOp) { defer wg.Done() - if err := fetchResource(fetchCtx, working, timeOut, fetchCfg.Log); err != nil { + + _, err := io.Copy(fetch.target, fetch.source) + if err != nil { errCh <- err } - }(res) + // Close asap + fetch.Close() + ops[index] = nil + }(i, res) } // Wait for all requests to be done @@ -115,7 +112,7 @@ func (fetchType) Prepare(ctx context.Context, capComm *rocket.CapComm, task rock // Close the error channel as no more errors close(errCh) - // Wait for err change to close + // Wait for err channel to close <-doneCh return err } @@ -123,79 +120,60 @@ func (fetchType) Prepare(ctx context.Context, capComm *rocket.CapComm, task rock return fn, nil } -func getResourcesList(ctx context.Context, capComm *rocket.CapComm, list []FetchResource) ([]urlFileTuple, error) { - resources := make([]urlFileTuple, 0, len(list)) - for index, res := range list { - tup := urlFileTuple{} - if out, err := capComm.ExpandString(ctx, "out", res.Output); err != nil { - return nil, errors.Wrapf(err, "expanding output %d", index) - } else if out == "" { - return nil, fmt.Errorf("output %d is blank", index) - } else { - tup.out = out +func getFetchOpsFromResourceList(ctx context.Context, capComm *rocket.CapComm, list []FetchResource) (fetchOps, error) { + resources := make(fetchOps, 0, len(list)) + for index, resource := range list { + op, err := getResource(ctx, capComm, resource) + if err != nil { + resources.Close() + return nil, errors.Wrapf(err, "resource[%d]", index) } - if url, err := capComm.ExpandString(ctx, "url", res.URL); err != nil { - return nil, errors.Wrapf(err, "expanding url %d", index) - } else if url == "" { - return nil, fmt.Errorf("url %d is blank", index) - } else { - tup.url = url - } - - resources = append(resources, tup) + resources = append(resources, op) } return resources, nil } -func getTimeOut(cfgTimeout *int) (time.Duration, error) { - var timeout time.Duration - if cfgTimeout != nil { - timeout = time.Second * time.Duration(*cfgTimeout) - } else { - timeout = time.Second * time.Duration(30) - } - if timeout < time.Second { - return timeout, fmt.Errorf("timeout %d is to short", timeout/time.Second) - } - - return timeout, nil -} - -func fetchResource(ctx context.Context, res urlFileTuple, timeOut time.Duration, log bool) error { - ctxTimeout, cancel := context.WithTimeout(ctx, timeOut) - defer cancel() - - req, err := http.NewRequest("GET", res.url, nil) +func getResource(ctx context.Context, capComm *rocket.CapComm, resource FetchResource) (*fetchOp, error) { + srcRp, err := capComm.InputSpecToResourceProvider(ctx, resource.Source) if err != nil { - return errors.Wrapf(err, "creating request for %s", res.url) + return nil, errors.Wrap(err, "source") } - resp, err := ctxhttp.Do(ctxTimeout, nil, req) + outRp, err := capComm.OutputSpecToResourceProvider(ctx, resource.Output) if err != nil { - return errors.Wrapf(err, "getting %s", res.url) + return nil, errors.Wrap(err, "output") } - defer resp.Body.Close() - if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { - // Bad response - return fmt.Errorf("response (%d) %s for %s", resp.StatusCode, resp.Status, res.url) + reader, err := srcRp.OpenRead(ctx) + if err != nil { + return nil, errors.Wrap(err, "source") } - // Create the file - out, err := os.Create(res.out) + writer, err := outRp.OpenWrite(ctx) if err != nil { - return errors.Wrapf(err, "creating %s", res.out) + reader.Close() + return nil, errors.Wrap(err, "output") } - defer out.Close() - // Write the body to file - bytes, err := io.Copy(out, resp.Body) - if log { - loggee.Infof("fetch %s => %s, %d bytes", res.url, res.out, bytes) + return &fetchOp{ + source: reader, + target: writer, + }, nil +} + +func (op fetchOp) Close() { + _ = op.source.Close() + _ = op.target.Close() +} + +func (ops fetchOps) Close() { + for _, op := range ops { + if op != nil { + op.Close() + } } - return err } func init() { diff --git a/pkg/rocket/builtin/redirect.go b/pkg/rocket/builtin/redirect.go deleted file mode 100644 index 9c4bd77..0000000 --- a/pkg/rocket/builtin/redirect.go +++ /dev/null @@ -1,154 +0,0 @@ -package builtin - -import ( - "bufio" - "os" - "os/exec" - - "github.com/nehemming/cirocket/pkg/loggee" - "github.com/nehemming/cirocket/pkg/rocket" - "github.com/pkg/errors" -) - -type ( - closeFiles []*os.File - - pipeFuncs func() chan struct{} -) - -func (cf closeFiles) Close() { - for _, c := range cf { - c.Close() - } -} - -type redirectWorker struct { - pipeWorkers []pipeFuncs - fileClosers closeFiles - isOutAndErrMerged bool -} - -func (redirect *redirectWorker) setUpStdIn(capComm *rocket.CapComm, cmd *exec.Cmd) error { - // Handle input - inFd := capComm.GetFileDetails(rocket.InputIO) - if inFd != nil { - inFile, err := inFd.OpenInput() - if err != nil { - return errors.Wrap(err, string(rocket.InputIO)) - } - redirect.fileClosers = append(redirect.fileClosers, inFile) - cmd.Stdin = inFile - } else { - cmd.Stdin = os.Stdin - } - - return nil -} - -func (redirect *redirectWorker) setUpStdOut(capComm *rocket.CapComm, cmd *exec.Cmd, logOutput bool) error { - // Handle output - outFd := capComm.GetFileDetails(rocket.OutputIO) - if outFd != nil { - outFile, err := outFd.OpenOutput() - if err != nil { - return errors.Wrap(err, string(rocket.OutputIO)) - } - redirect.fileClosers = append(redirect.fileClosers, outFile) - cmd.Stdout = outFile - - // Check if error is merged - if outFd.InMode(rocket.IOModeError) { - redirect.isOutAndErrMerged = true - cmd.Stderr = outFile - } - } else if logOutput { - outPipe, err := cmd.StdoutPipe() - if err != nil { - return errors.Wrap(err, string(rocket.OutputIO)) - } - - fn := func() chan struct{} { - ch := make(chan struct{}) - - go func() { - defer close(ch) - in := bufio.NewScanner(outPipe) - - for in.Scan() { - loggee.Info(in.Text()) // write each line to your log, or anything you need - } - }() - - return ch - } - - redirect.pipeWorkers = append(redirect.pipeWorkers, fn) - } else { - // redirect to std out - cmd.Stdout = os.Stdout - } - - return nil -} - -func (redirect *redirectWorker) setUpStdErr(capComm *rocket.CapComm, cmd *exec.Cmd, logOutput bool) error { - errFd := capComm.GetFileDetails(rocket.ErrorIO) - if errFd != nil { - errFile, err := errFd.OpenOutput() - if err != nil { - return errors.Wrap(err, string(rocket.ErrorIO)) - } - redirect.fileClosers = append(redirect.fileClosers, errFile) - cmd.Stderr = errFile - } else if logOutput { - errPipe, err := cmd.StderrPipe() - if err != nil { - return errors.Wrap(err, string(rocket.OutputIO)) - } - - fn := func() chan struct{} { - ch := make(chan struct{}) - - go func() { - defer close(ch) - in := bufio.NewScanner(errPipe) - - for in.Scan() { - loggee.Warn(in.Text()) // write each line to your log, or anything you need - } - }() - - return ch - } - - redirect.pipeWorkers = append(redirect.pipeWorkers, fn) - - } else { - cmd.Stderr = os.Stderr - } - - return nil -} - -func setupRedirect(capComm *rocket.CapComm, cmd *exec.Cmd, runCfg *Run) ([]pipeFuncs, closeFiles, error) { - redirect := &redirectWorker{ - pipeWorkers: make([]pipeFuncs, 0, 3), - fileClosers: make(closeFiles, 0, 3), - } - - if err := redirect.setUpStdIn(capComm, cmd); err != nil { - return redirect.pipeWorkers, redirect.fileClosers, err - } - - if err := redirect.setUpStdOut(capComm, cmd, runCfg.LogOutput); err != nil { - return redirect.pipeWorkers, redirect.fileClosers, err - } - - if !redirect.isOutAndErrMerged { - if err := redirect.setUpStdErr(capComm, cmd, !runCfg.DirectError); err != nil { - return redirect.pipeWorkers, redirect.fileClosers, err - } - } - - return redirect.pipeWorkers, redirect.fileClosers, nil -} diff --git a/pkg/rocket/builtin/run.go b/pkg/rocket/builtin/run.go index cbe7286..ab6bc67 100644 --- a/pkg/rocket/builtin/run.go +++ b/pkg/rocket/builtin/run.go @@ -32,14 +32,6 @@ type ( // Redirect handles input and output redirection. rocket.Redirection `mapstructure:",squash"` - - // LogOutput if true will cause output to be logged rather than going to go to std output. - // If an output file is specified it will be used instead. - LogOutput bool `mapstructure:"logStdOut"` - - // DirectError when true causes the commands std error output to go direct to running processes std error - // When DirectError is false std error output is logged. - DirectError bool `mapstructure:"directStdErr"` } runType struct{} @@ -56,13 +48,13 @@ func (runType) Prepare(ctx context.Context, capComm *rocket.CapComm, task rocket return nil, errors.Wrap(err, "parsing template type") } - // Get the command line - commandLine, err := getCommandLine(ctx, capComm, runCfg) - if err != nil { - return nil, err - } - fn := func(execCtx context.Context) error { + // Get the command line + commandLine, err := getCommandLine(ctx, capComm, runCfg) + if err != nil { + return err + } + // Setup command cmd := exec.Command(commandLine.ProgramPath, commandLine.Args...) cmd.Env = capComm.GetExecEnv() @@ -74,7 +66,7 @@ func (runType) Prepare(ctx context.Context, capComm *rocket.CapComm, task rocket // Run command var runExitCode int - err := runCmd(execCtx, capComm, runCfg, cmd) + err = runCmd(execCtx, capComm, cmd) if err != nil { // Issue caught if exitError, ok := err.(*exec.ExitError); ok { @@ -136,11 +128,36 @@ func startProcessSignalHandlee(ctx context.Context, cmd *exec.Cmd) chan struct{} return done } -func runCmd(ctx context.Context, capComm *rocket.CapComm, runCfg *Run, cmd *exec.Cmd) error { - pipeFuncs, cf, err := setupRedirect(capComm, cmd, runCfg) - defer cf.Close() // close any files we opened in redirect - if err != nil { - return err +func runCmd(ctx context.Context, capComm *rocket.CapComm, cmd *exec.Cmd) error { + inputResource := capComm.GetResource(rocket.InputIO) + outputResource := capComm.GetResource(rocket.OutputIO) + errorResource := capComm.GetResource(rocket.ErrorIO) + + if inputResource != nil { + stdIn, err := inputResource.OpenRead(ctx) + if err != nil { + return errors.Wrap(err, "input") + } + cmd.Stdin = stdIn + defer stdIn.Close() + } + + if outputResource != nil { + stdOut, err := outputResource.OpenWrite(ctx) + if err != nil { + return errors.Wrap(err, "output") + } + cmd.Stdout = stdOut + defer stdOut.Close() + } + + if errorResource != nil { + stdErr, err := errorResource.OpenWrite(ctx) + if err != nil { + return errors.Wrap(err, "error") + } + cmd.Stderr = stdErr + defer stdErr.Close() } // Start the process @@ -152,17 +169,6 @@ func runCmd(ctx context.Context, capComm *rocket.CapComm, runCfg *Run, cmd *exec signalHandlerDoneChannel := startProcessSignalHandlee(ctx, cmd) defer close(signalHandlerDoneChannel) - // Handle pipes - channels := make([]chan struct{}, 0, len(pipeFuncs)) - for _, fn := range pipeFuncs { - channels = append(channels, fn()) - } - - // Wait for channels to close, meaning pipe has shut - for _, ch := range channels { - <-ch - } - // Wait for process exit return cmd.Wait() } diff --git a/pkg/rocket/builtin/runinit_test.go b/pkg/rocket/builtin/runinit_test.go new file mode 100644 index 0000000..b68c2fc --- /dev/null +++ b/pkg/rocket/builtin/runinit_test.go @@ -0,0 +1,23 @@ +package builtin + +import ( + "context" + "testing" + + "github.com/nehemming/cirocket/pkg/loggee" + "github.com/nehemming/cirocket/pkg/loggee/stdlog" + "github.com/nehemming/cirocket/pkg/rocket" +) + +func TestRunInit(t *testing.T) { + loggee.SetLogger(stdlog.New()) + + mc := rocket.NewMissionControl() + RegisterAll(mc) + + mission, cfgFile := loadMission("init_output") + + if err := mc.LaunchMission(context.Background(), cfgFile, mission); err != nil { + t.Error("failure", err) + } +} diff --git a/pkg/rocket/builtin/template.go b/pkg/rocket/builtin/template.go index ad82029..133bf20 100644 --- a/pkg/rocket/builtin/template.go +++ b/pkg/rocket/builtin/template.go @@ -2,16 +2,17 @@ package builtin import ( "context" - "os" + "io" "text/template" "github.com/mitchellh/mapstructure" + "github.com/nehemming/cirocket/pkg/providers" "github.com/nehemming/cirocket/pkg/rocket" "github.com/pkg/errors" ) const ( - templateFileTag = rocket.NamedIO("template") + templateResourceID = providers.ResourceID("template") ) type ( @@ -20,11 +21,14 @@ type ( // Delims is used to change the standard golang templatiing delimiters // This can be useful when processing a source file that itself uses golang tempalting. Template struct { - // Template file - FileTemplate string `mapstructure:"template"` - InlineTemplate string `mapstructure:"inline"` - rocket.OutputSpec `mapstructure:",squash"` - Delims rocket.Delims `mapstructure:"delims"` + Template rocket.InputSpec `mapstructure:"template"` + + // OutputSpec is the specification for the template output + Output *rocket.OutputSpec `mapstructure:"output"` + + // Delims are the delimiters used to identify template script. + // Leave blank for the default go templating delimiters + Delims rocket.Delims `mapstructure:"delims"` } templateType struct{} @@ -34,25 +38,20 @@ func (templateType) Type() string { return "template" } -func validateTemplateConfig(ctx context.Context, capComm *rocket.CapComm, templateCfg *Template) error { - if templateCfg.FileTemplate != "" && templateCfg.InlineTemplate != "" { - return errors.New("both a file and inline template have been specified, only one is allowed") - } else if templateCfg.FileTemplate == "" && templateCfg.InlineTemplate == "" { - return errors.New("neither a file or inline template have been specified, please provide one of them") +func configureSources(ctx context.Context, capComm *rocket.CapComm, templateCfg *Template) error { + // Preevent inline being expanded as input to a template + if templateCfg.Template.Inline != "" { + templateCfg.Template.SkipExpand = true } - - // Expand the template file name - if templateCfg.FileTemplate != "" { - if err := capComm.AddFile(ctx, templateFileTag, templateCfg.FileTemplate, rocket.IOModeInput); err != nil { - return errors.Wrap(err, "expanding template file name") - } + if err := capComm.AttachInputSpec(ctx, templateResourceID, templateCfg.Template); err != nil { + return errors.Wrap(err, "template") } - // Expand redirect settings into cap Comm - if err := capComm.AttachOutput(ctx, templateCfg.OutputSpec); err != nil { - return errors.Wrap(err, "expanding output settings") + if templateCfg.Output != nil { + if err := capComm.AttachOutputSpec(ctx, rocket.OutputIO, *templateCfg.Output); err != nil { + return errors.Wrap(err, "output") + } } - return nil } @@ -63,25 +62,29 @@ func (templateType) Prepare(ctx context.Context, capComm *rocket.CapComm, task r return nil, errors.Wrap(err, "parsing template type") } - if err := validateTemplateConfig(ctx, capComm, templateCfg); err != nil { - return nil, err - } - fn := func(runCtx context.Context) error { + // Late configure sources, allow previous steps to be available + if err := configureSources(runCtx, capComm, templateCfg); err != nil { + return err + } + // Load the template - t, err := loadTemplate(capComm, task.Name, templateCfg) + t, err := loadTemplate(runCtx, capComm, task.Name, templateCfg) if err != nil { - return errors.Wrap(err, "create template") + return errors.Wrap(err, "template") } - writer, cf, err := setupOutput(capComm) - defer cf.Close() + // Prepare output + outputResource := capComm.GetResource(rocket.OutputIO) + writer, err := outputResource.OpenWrite(runCtx) if err != nil { - return err + return errors.Wrap(err, "output") } + defer writer.Close() + // Execute if err := t.Execute(writer, capComm.GetTemplateData(ctx)); err != nil { - return errors.Wrap(err, "execute template") + return errors.Wrap(err, "template") } return nil @@ -90,44 +93,23 @@ func (templateType) Prepare(ctx context.Context, capComm *rocket.CapComm, task r return fn, nil } -func loadTemplate(capComm *rocket.CapComm, name string, templateCfg *Template) (*template.Template, error) { - var tt string - if templateCfg.InlineTemplate != "" { - tt = templateCfg.InlineTemplate - } else if b, err := capComm.GetFileDetails(templateFileTag).ReadFile(); err != nil { - return nil, errors.Wrap(err, "read template file") - } else { - tt = string(b) - } - - d := templateCfg.Delims - - t, err := template.New(name). - Funcs(capComm.FuncMap()). - Delims(d.Left, d.Right). - Parse(tt) +func loadTemplate(ctx context.Context, capComm *rocket.CapComm, name string, templateCfg *Template) (*template.Template, error) { + // Get template data + templateResource := capComm.GetResource(templateResourceID) + r, err := templateResource.OpenRead(ctx) if err != nil { - return nil, errors.Wrap(err, "parse template") + return nil, err } + defer r.Close() - return t, err -} - -func setupOutput(capComm *rocket.CapComm) (*os.File, closeFiles, error) { - cf := make(closeFiles, 0, 3) - - // Handle output - outFd := capComm.GetFileDetails(rocket.OutputIO) - if outFd != nil { - outFile, err := outFd.OpenOutput() - if err != nil { - return nil, cf, errors.Wrap(err, string(rocket.OutputIO)) - } - cf = append(cf, outFile) - return outFile, cf, nil + b, err := io.ReadAll(r) + if err != nil { + return nil, err } - return os.Stdout, cf, nil + return template.New(name). + Funcs(capComm.FuncMap()). + Delims(templateCfg.Delims.Left, templateCfg.Delims.Right).Parse(string(b)) } func init() { diff --git a/pkg/rocket/builtin/testdata/badfetch.yml b/pkg/rocket/builtin/testdata/badfetch.yml index 2fa4bcf..b2e6250 100644 --- a/pkg/rocket/builtin/testdata/badfetch.yml +++ b/pkg/rocket/builtin/testdata/badfetch.yml @@ -6,8 +6,12 @@ stages: name: fetch test data log: true resources: - - url: 'https://raw.githubusercontent.com/nehemming/oauthproxy/master/README.mdd' - output: 'testdata/readme.tmp' - - url: 'https://raw.githubusercontent.com/nehemming/oauthproxy/master/README.md' - output: 'testdata/readme2.tmp' + - source: + url: 'https://raw.githubusercontent.com/nehemming/oauthproxy/master/README.mdd' + output: + path: 'testdata/readme.tmp' + - source: + url: 'https://raw.githubusercontent.com/nehemming/oauthproxy/master/README.md' + output: + path: 'testdata/readme2.tmp' \ No newline at end of file diff --git a/pkg/rocket/builtin/testdata/fetch.yml b/pkg/rocket/builtin/testdata/fetch.yml index 26e1617..fba0d87 100644 --- a/pkg/rocket/builtin/testdata/fetch.yml +++ b/pkg/rocket/builtin/testdata/fetch.yml @@ -5,6 +5,8 @@ stages: name: fetch test data log: true resources: - - url: 'https://raw.githubusercontent.com/nehemming/oauthproxy/master/README.md' - output: 'testdata/readme.tmp' + - source: + url: 'https://raw.githubusercontent.com/nehemming/oauthproxy/master/README.md' + output: + path: 'testdata/readme.tmp' \ No newline at end of file diff --git a/pkg/rocket/builtin/testdata/hello.yml b/pkg/rocket/builtin/testdata/hello.yml index d45e890..28e3e89 100644 --- a/pkg/rocket/builtin/testdata/hello.yml +++ b/pkg/rocket/builtin/testdata/hello.yml @@ -8,13 +8,16 @@ stages: - tasks: - type: template name: sayhello - inline: | - Hello World {{.person}} + template: + inline: | + Hello World {{.person}} - type: template name: list file delims: left: '[[' right: ']]' - template: "testdata/templtetest.txt" - output: "testdata/{{.person}}.tmp" + template: + path: "testdata/templtetest.txt" + output: + path: "testdata/{{.person}}.tmp" diff --git a/pkg/rocket/builtin/testdata/init_output.yml b/pkg/rocket/builtin/testdata/init_output.yml new file mode 100644 index 0000000..50ffbfa --- /dev/null +++ b/pkg/rocket/builtin/testdata/init_output.yml @@ -0,0 +1,174 @@ +# Sample configuration file +# Edit this file to create a new mission + +# configuration values often support go template expansion using the {{ }} delimiters. Any expansion should be specified in quote marks to form valid yaml. + +# All missions can be named. If a name is not specified it is take from the base name of the configuration file. +name: "init sample mission" + +# version is optional but if provided should vbe version 1.0 +version: '1.0' + +# sequences can be used to specify what stages are executed. If they are not included all stages are run in file order. +# when one or more sequences are defined the stages are run in the order of the requested sequence(s). +# in the cas below the command would be: cirocket launch run +# sequences: +# run: +# - "sample stage" + +# params are configuration settings. The are accessible in all settings that support template expansion +# params can be specified as part of the mission, stage or task. Params are inherited and can be overbidden +# all params must have aa name, the key by which they are accessible name: fred is accessible as {{ .fred }} in templates. +# param values can be specified using the value tag or read from a file or web url. Both are subject to template expansion. +# If both value and file are specified the final value is the concatenation of the value property and file contents. +# skipExpand stops template expansion occurring. If optional is true no error will be raised if the file or url is not found +params: + - name: welcome + value: hello world + #- name: secret + # path: some_file.txt + # url: "https//..." + # skipExpand: false + # optional: false + +# env contains environment variables that are defined for use in template expansion or passed to any sub processes executed +# env variables are subject to param expansion, so can for example be passed secrets from params. +# env variables are accessible in template expansion as either ${var} or {{ .Env.var }} +# all environment variables accessible to the host process are added to the default collection inherited by the mission. +# env sections can be added at the mission, stage and task levels. +# by default stages and tasks inherit their parents environment variables. If this is undesirable, for example you do not want +# to run a sub process with API keys, these can either be overrides as blanks or the task/stage definition can include the +# noTrust: true setting. This prevents inheritance. +env: + THRUSTERS: go + +# missions are broken down into stages, each stage contains a set of zero or more tasks. +# how stages are processed depends on the presence oor absence of the sequences section. +# If no sequences section is provided, stages are executed in the order they are defined in this file. +# With a sequence file present the order and what is executed is determined by the sequence +# each stage may have a name, and may contain stage level param and env(ironment) variables. +stages: + - name: "sample stage" + # env: + # param: + # noTrust: true + + # a filter section can be added to tasks and stages, this limits the running of the step to + # host applications running on specific operating systems or architectures. This can be useful + # if task have windows or linux specific scripts etc. Values used for filtering come form the go + # architecture and os names ... i.e. macos is known as darwin. + # filter: + # includeOS: + # - windows + # includeArch: + # - amd64 + # excludeOS: + # - linux + # excludeArch: + # - i386 + + # tasks are defined under a stage. tasks represent a activity of a specific type + # types currently supported include 'run' and 'template' Others may be added shortly. + # each task type may have type specific properties, however all share the properties (env,params, filter and noTrust) + # tasks can optionally have a name, other wise their name is task+ordinal position in list. + tasks: + - name: get the go version + # env: + # param: + # noTrust: true + # filter: + + type: run + # run specific settings + # command is the command to run. It may include command line arguments + # in the example below command: go version is equally valid + # args are appended to the arg list extracted from the command line + # command line and args can use templates + # glob specifies if args should be expanded linux shell style or passed as is to the program. + # to avoid globing wildcards can be enclosed in quotes. + command: go + args: + - version + # glob: false + # logStdOut: false + + # sub process output is either sent to a file, the log or to the host applications stdout. + # redirection uses the input, output and error sub keys + # output allows a file or variable to be specified, append controls if the file should be appended to or truncated + # logStdOut sends log output, if not going to a file to the log, otherwise its send to stdout. + # output: + # path: filename + # variable: exported_variable + # append: false + # logStdOut: true + + # a input file can be specified in place of stdin, sources include variables, files, urls or inline + # all arguments support template expansion. This can be disabled with the skipExpand arg. The timeout setting + # allows a request timeout limit to be specified. If blank default is 30 seconds. + # input: + # inline: test + # path: file + # url: url + # variable: exported_variable + + # error output normally goes to the log, this can be modified using the error sub key + # it can be directed to a file, merged with the output file or direct to the host processes stderr. + # error: + # path: filename + # variable: exported_variable + # append: false + # directStdErr: false + + # template tasks are used to run a go template + - name: template_example + type: template + + # template input is defined by the template sub key and its output by the output key + # inline templates are not expanded prior to template processing + # template: + # inline: test + # path: file + # url: url + # variable: exported_variable + + # output: + # path: filename + # variable: exported_variable + # append: false + + # either a template or inline property must be specified. + # template is the name of the go template file + # inline is an inline expansion + #template: file + template: + inline: | + Say {{.welcome}}! + + # template output is either sent to a file or to the host applications stdout. + # output: filename + # appendOutput: true + + # cleaner tasks are used to delete files or directories + # param expansion can be used and the file specs perform globing + - name: cleaner_task + type: cleaner + log: true + # files: + # - 'file-1' + + + # fetch pulls data from files, inline statements and urls into exported variables or local files. + - name: fetch_task + type: fetch + # resources: + # - source: + # inline: test + # path: file + # url: url + # variable: exported_variable + # output: + # path: 'testdata/readme.tmp' + # variable: exported_variable + # append: false + +# end of file diff --git a/pkg/rocket/builtin/testdata/rungo.yml b/pkg/rocket/builtin/testdata/rungo.yml index 56bd0ec..43b627e 100644 --- a/pkg/rocket/builtin/testdata/rungo.yml +++ b/pkg/rocket/builtin/testdata/rungo.yml @@ -14,4 +14,5 @@ stages: command: go args: - version - output: "testdata/out.tmp" + output: + path: "testdata/out.tmp" diff --git a/pkg/rocket/capcomm.go b/pkg/rocket/capcomm.go index 257d496..568c79b 100644 --- a/pkg/rocket/capcomm.go +++ b/pkg/rocket/capcomm.go @@ -4,14 +4,20 @@ import ( "bytes" "context" "fmt" + "io" + "net/http" "os" "path/filepath" "runtime" "strings" "text/template" + "time" "github.com/nehemming/cirocket/pkg/buildinfo" + "github.com/nehemming/cirocket/pkg/loggee" + "github.com/nehemming/cirocket/pkg/providers" "github.com/pkg/errors" + "golang.org/x/net/context/ctxhttp" ) const ( @@ -28,6 +34,9 @@ const ( // BuildTag is the data template key for build information about the hosting application. BuildTag = "Build" + // VariableTag is the top key for variables in the template data. + VariableTag = "Var" + // AdditionalMissionTag is the data template key to additional mission information. AdditionalMissionTag = "Additional" @@ -67,16 +76,22 @@ type ( runtime Runtime sealed bool mission *Mission - ioSettings *ioSettings + resources providers.ResourceProviderMap + variables exportMap + exportTo exportMap + log loggee.Logger } + + exportMap map[string]string ) -func setParamsFromConfigFile(kv map[string]string, configFile string) { +// setParamsFromConfigFile adds config file entries to the environment map. +func setParamsFromConfigFile(env map[string]string, configFile string) { dir, file := filepath.Split(configFile) - kv[ConfigFileFullPath], _ = filepath.Abs(configFile) - kv[ConfigFile] = file - kv[ConfigBaseName] = strings.TrimSuffix(file, filepath.Ext(file)) - kv[ConfigDir] = strings.TrimSuffix(dir, string(filepath.Separator)) + env[ConfigFileFullPath], _ = filepath.Abs(configFile) + env[ConfigFile] = file + env[ConfigBaseName] = strings.TrimSuffix(file, filepath.Ext(file)) + env[ConfigDir] = strings.TrimSuffix(dir, string(filepath.Separator)) } func initFuncMap() template.FuncMap { @@ -100,7 +115,7 @@ func initFuncMap() template.FuncMap { } // newCapCommFromEnvironment creates a new capCom from the environment. -func newCapCommFromEnvironment(configFile string) *CapComm { +func newCapCommFromEnvironment(configFile string, log loggee.Logger) *CapComm { paramKvg := NewKeyValueGetter(nil) setParamsFromConfigFile(paramKvg.kv, configFile) @@ -114,8 +129,20 @@ func newCapCommFromEnvironment(configFile string) *CapComm { GOOS: runtime.GOOS, GOARCH: runtime.GOARCH, }, - ioSettings: newIOSettings(), + resources: make(providers.ResourceProviderMap), + variables: make(exportMap), + log: log, } + + // cc.resources[InputIO] = providers.NewNonClosingReaderProvider(os.Stdin) + cc.resources[Stdin] = providers.NewNonClosingReaderProvider(os.Stdin) + + cc.resources[OutputIO] = providers.NewNonClosingWriterProvider(os.Stdout) + cc.resources[Stdout] = providers.NewNonClosingWriterProvider(os.Stdout) + cc.resources[Stderr] = providers.NewNonClosingWriterProvider(os.Stderr) + + cc.resources[ErrorIO] = providers.NewLogProvider(log, providers.LogWarn) + return cc } @@ -133,7 +160,10 @@ func (capComm *CapComm) Copy(noTrust bool) *CapComm { GOOS: runtime.GOOS, GOARCH: runtime.GOARCH, }, - ioSettings: capComm.ioSettings.newCopy(), + resources: capComm.resources.Copy(), + variables: make(exportMap), + exportTo: capComm.variables, + log: capComm.log, } // Non trusted CapComm copies do not receive environment variables from their parent @@ -163,6 +193,16 @@ func (capComm *CapComm) Seal() *CapComm { return capComm } +// Log returns the cap com logger. +func (capComm *CapComm) Log() loggee.Logger { + return capComm.log +} + +func (capComm *CapComm) ExportVariable(key, value string) *CapComm { + capComm.exportTo[key] = value + return capComm +} + // WithMission attaches the mission to the CapComm. func (capComm *CapComm) WithMission(mission *Mission) *CapComm { capComm.mustNotBeSealed() @@ -205,57 +245,290 @@ func (capComm *CapComm) AddAdditionalMissionData(missionData TemplateData) *CapC return capComm } -// AddFile adds a key named file specificsation into the CapComm. +// AddResource adds a resource to the capComm object. +func (capComm *CapComm) AddResource(name providers.ResourceID, provider providers.ResourceProvider) { + capComm.resources[name] = provider +} + +// GetResource a resource. +func (capComm *CapComm) GetResource(name providers.ResourceID) providers.ResourceProvider { + return capComm.resources[name] +} + +// AddFileResource adds a key named file specificsation into the CapComm. // The filePath follows standard template and environment variable expansion. The mode controls how the file will be used. // Files can be added to sealed CapComm's. -func (capComm *CapComm) AddFile(ctx context.Context, name NamedIO, filePath string, mode IOMode) error { - v, err := capComm.ExpandString(ctx, string(name), filePath) +func (capComm *CapComm) AddFileResource(ctx context.Context, name providers.ResourceID, filePath string, mode providers.IOMode) error { + path, err := capComm.ExpandString(ctx, string(name), filePath) if err != nil { return errors.Wrapf(err, "expand %s file path", name) } - capComm.ioSettings.addFilePath(name, v, mode) + + provider, err := providers.NewFileProvider(path, mode, 0666, false) + if err != nil { + return err + } + capComm.resources[name] = provider + return nil } -// GetFileDetails returns the file details of the named file. If the file key does not exist nil is returned. -func (capComm *CapComm) GetFileDetails(name NamedIO) *FileDetail { - return capComm.ioSettings.getFileDetails(name) +func validateInputSpec(inputSpec *InputSpec) error { + count := 0 + if inputSpec.Variable != "" { + count++ + } + if inputSpec.Inline != "" { + count++ + } + if inputSpec.Path != "" { + count++ + } + if inputSpec.URL != "" { + count++ + } + + if count > 1 { + return errors.New("more than one input source was specified, only one is permitted") + } + if count == 0 { + return errors.New("no input source was specified") + } + return nil } -// AttachOutput attaches an output specification to a capComm. -func (capComm *CapComm) AttachOutput(ctx context.Context, redirect OutputSpec) error { - // Handle output +func (capComm *CapComm) createProviderFromInputSpec(ctx context.Context, inputSpec InputSpec) (providers.ResourceProvider, error) { //nolint:cyclop + var rp providers.ResourceProvider + var err error + var v string + if inputSpec.Variable != "" { + v, ok := capComm.exportTo[inputSpec.Variable] + if !ok && !inputSpec.Optional { + return nil, fmt.Errorf("variable %s not found", inputSpec.Variable) + } + rp = providers.NewNonClosingReaderProvider(bytes.NewBufferString(v)) + } else if inputSpec.Inline != "" { + if !inputSpec.SkipExpand { + v, err = capComm.ExpandString(ctx, "inline", inputSpec.Inline) + } else { + v = inputSpec.Inline + } + rp = providers.NewNonClosingReaderProvider(bytes.NewBufferString(v)) + } else if inputSpec.Path != "" { + if !inputSpec.SkipExpand { + v, err = capComm.ExpandString(ctx, "path", inputSpec.Path) + if err != nil { + return nil, err + } + } else { + v = inputSpec.Path + } + rp, err = providers.NewFileProvider(v, providers.IOModeInput, 0, inputSpec.Optional) + } else if inputSpec.URL != "" { + if !inputSpec.SkipExpand { + v, err = capComm.ExpandString(ctx, "url", inputSpec.URL) + if err != nil { + return nil, err + } + } else { + v = inputSpec.URL + } + rp, err = providers.NewURLProvider(v, time.Second*time.Duration(inputSpec.URLTimeout), inputSpec.Optional) + } else { + panic("validation bad input spec") + } + + return rp, err +} - if redirect.Output == "" { - return nil +func (capComm *CapComm) InputSpecToResourceProvider(ctx context.Context, inputSpec InputSpec) (providers.ResourceProvider, error) { + if err := validateInputSpec(&inputSpec); err != nil { + return nil, err } - mode := IOModeOutput - if redirect.AppendOutput { - mode |= IOModeAppend + return capComm.createProviderFromInputSpec(ctx, inputSpec) +} + +// AttachInputSpec adds a named input spec to the capCom resources. +func (capComm *CapComm) AttachInputSpec(ctx context.Context, name providers.ResourceID, inputSpec InputSpec) error { + if name == "" { + return errors.New("name cannot be blank") + } + + rp, err := capComm.InputSpecToResourceProvider(ctx, inputSpec) + + if err == nil { + capComm.AddResource(name, rp) + } + + return err +} + +func validateOutputSpec(outputSpec *OutputSpec) error { + count := 0 + if outputSpec.Variable != "" { + count++ + } + if outputSpec.Path != "" { + count++ + } + + if count > 1 { + return errors.New("more than one output source was specified, only one is permitted") + } + if count == 0 { + return errors.New("no output source was specified") + } + + return nil +} + +func (capComm *CapComm) createProviderFromOutputSpec(ctx context.Context, outputSpec OutputSpec, mode providers.IOMode) (providers.ResourceProvider, error) { + if outputSpec.Variable != "" { + // Write to a variable + return newVariableWriter(capComm, outputSpec.Variable), nil + } + + // Are we appending? + if outputSpec.Append { + mode |= providers.IOModeAppend + } else { + mode |= providers.IOModeTruncate + } + + // Get the file mode + var fileMode os.FileMode + if outputSpec.FileMode == 0 { + fileMode = 0666 } else { - mode |= IOModeTruncate + fileMode = os.FileMode(outputSpec.FileMode) } - return capComm.AddFile(ctx, OutputIO, redirect.Output, mode) + // Expand the value + var v string + var err error + if !outputSpec.SkipExpand { + v, err = capComm.ExpandString(ctx, "path", outputSpec.Path) + if err != nil { + return nil, err + } + } else { + v = outputSpec.Path + } + + return providers.NewFileProvider(v, mode, fileMode, false) +} + +func (capComm *CapComm) OutputSpecToResourceProvider(ctx context.Context, outputSpec OutputSpec) (providers.ResourceProvider, error) { + err := validateOutputSpec(&outputSpec) + if err != nil { + return nil, err + } + + return capComm.createProviderFromOutputSpec(ctx, outputSpec, providers.IOModeOutput) +} + +// AttachOutputSpec attaches an output specification to the capComm. +func (capComm *CapComm) AttachOutputSpec(ctx context.Context, name providers.ResourceID, outputSpec OutputSpec) error { + if name == "" { + return errors.New("name cannot be blank") + } + + rp, err := capComm.OutputSpecToResourceProvider(ctx, outputSpec) + + if err == nil { + capComm.AddResource(name, rp) + } + + return err +} + +func validateRedirection(redirect *Redirection) error { //nolint + if redirect.LogOutput && redirect.Output != nil { + return errors.New("cannot both redirect to the log and also provide an output specification") + } + if redirect.DirectError && redirect.Error != nil { + return errors.New("cannot both redirect to stderr and also provide an error specification") + } + if redirect.MergeErrorWithOutput && redirect.Error != nil { + return errors.New("cannot merge errors with output and specify an error specification") + } + if redirect.Input != nil { + if err := validateInputSpec(redirect.Input); err != nil { + return errors.Wrap(err, string(InputIO)) + } + } + if redirect.Output != nil { + err := validateOutputSpec(redirect.Output) + if err != nil { + return errors.Wrap(err, string(OutputIO)) + } + } + if redirect.Error != nil { + err := validateOutputSpec(redirect.Error) + if err != nil { + return errors.Wrap(err, string(ErrorIO)) + } + } + return nil } // AttachRedirect attaches a redirection specification to the capComm // Redirection covers in, out and error streams. -func (capComm *CapComm) AttachRedirect(ctx context.Context, redirect Redirection) error { - // Handle Input - if redirect.Input != "" { - if err := capComm.AddFile(ctx, InputIO, redirect.Input, IOModeInput); err != nil { +func (capComm *CapComm) AttachRedirect(ctx context.Context, redirect Redirection) error { //nolint + // Pre validate + if err := validateRedirection(&redirect); err != nil { + return err + } + + if redirect.Input != nil { + rp, err := capComm.createProviderFromInputSpec(ctx, *redirect.Input) + if err != nil { return err } + + capComm.AddResource(InputIO, rp) } - // Handle output - if err := capComm.attachRedirectOutput(ctx, redirect); err != nil { - return err + if redirect.LogOutput { + rp := providers.NewLogProvider(capComm.log, providers.LogInfo) + capComm.AddResource(OutputIO, rp) + + if redirect.MergeErrorWithOutput { + capComm.AddResource(ErrorIO, rp) + } + } + + if redirect.DirectError { + capComm.AddResource(ErrorIO, capComm.GetResource(Stderr)) } - return capComm.attachRedirectError(ctx, redirect) + if redirect.Output != nil { + mode := providers.IOModeOutput + if redirect.MergeErrorWithOutput { + mode |= providers.IOModeError + } + + rp, err := capComm.createProviderFromOutputSpec(ctx, *redirect.Output, mode) + if err != nil { + return err + } + capComm.AddResource(OutputIO, rp) + + if redirect.MergeErrorWithOutput { + capComm.AddResource(ErrorIO, rp) + } + } + + if redirect.Error != nil { + rp, err := capComm.createProviderFromOutputSpec(ctx, *redirect.Error, providers.IOModeError) + if err != nil { + return err + } + capComm.AddResource(ErrorIO, rp) + } + + return nil } // MergeParams adds params into an unsealed CapComm instance. @@ -319,6 +592,7 @@ func (capComm *CapComm) ExpandString(ctx context.Context, name, value string) (s // Create the template template, err := template.New(name). + Option("missingkey=zero"). Funcs(capComm.funcMap). Parse(value) if err != nil { @@ -381,6 +655,9 @@ func (capComm *CapComm) GetTemplateData(ctx context.Context) TemplateData { // Env data[EnvTag] = capComm.env.All() + // Add in variables + data[VariableTag] = capComm.variables.All() + if capComm.entrustedParentEnv != nil { data[ParentEnvTag] = capComm.entrustedParentEnv.All() } @@ -406,21 +683,54 @@ func (capComm *CapComm) expandShellEnv(value string) string { }) } +func getParamFromURL(ctx context.Context, url string, optional bool) (string, error) { + ctxTimeout, cancel := context.WithTimeout(ctx, timeOut) + defer cancel() + + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return "", errors.Wrapf(err, "creating request for %s", url) + } + + resp, err := ctxhttp.Do(ctxTimeout, nil, req) + if err != nil { + return "", errors.Wrapf(err, "getting %s", url) + } + defer resp.Body.Close() + + // Support optional gets + if optional && resp.StatusCode == http.StatusNotFound { + return "", nil + } + + if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { + // Bad response + return "", fmt.Errorf("response (%d) %s for %s", resp.StatusCode, resp.Status, url) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", err + } + + return string(body), nil +} + // expandParam carries out template expansion of a parameter. func (capComm *CapComm) expandParam(ctx context.Context, param Param) (string, error) { // Read param value := param.Value // If param has a file name, open it - if param.File != "" { - fileName, err := capComm.ExpandString(ctx, param.Name, param.File) + if param.Path != "" { + fileName, err := capComm.ExpandString(ctx, param.Name, param.Path) if err != nil { return "", errors.Wrap(err, "rexpanding file name") } b, err := os.ReadFile(fileName) if err != nil { - if !os.IsNotExist(err) || !param.FileOptional { + if !os.IsNotExist(err) || !param.Optional { return "", errors.Wrap(err, "reading value from file") } } else { @@ -428,8 +738,18 @@ func (capComm *CapComm) expandParam(ctx context.Context, param Param) (string, e } } + if param.URL != "" { + // pull the data from the url + body, err := getParamFromURL(ctx, param.URL, param.Optional) + if err != nil { + return "", err + } + + value += body + } + // Skip expanding a param - if param.SkipTemplate { + if param.SkipExpand { return value, nil } @@ -502,51 +822,6 @@ func (capComm *CapComm) mustNotBeSealed() { } } -func (capComm *CapComm) attachRedirectError(ctx context.Context, redirect Redirection) error { - // Handle error files - if !redirect.MergeErrorWithOutput && redirect.Error != "" { - mode := IOModeError - if redirect.AppendError { - mode |= IOModeAppend - } else { - mode |= IOModeTruncate - } - - if err := capComm.AddFile(ctx, ErrorIO, redirect.Error, mode); err != nil { - return err - } - } - - return nil -} - -func (capComm *CapComm) attachRedirectOutput(ctx context.Context, redirect Redirection) error { - if redirect.Output != "" { - mode := IOModeOutput - if redirect.AppendOutput { - mode |= IOModeAppend - } else { - mode |= IOModeTruncate - } - - // MergeErrorWithOutput uses the output spec for both out and error files. - if redirect.MergeErrorWithOutput { - mode |= IOModeError - - if err := capComm.AddFile(ctx, OutputIO, redirect.Output, mode); err != nil { - return err - } - - if err := capComm.ioSettings.duplicate(OutputIO, ErrorIO); err != nil { - return err - } - } else if err := capComm.AddFile(ctx, OutputIO, redirect.Output, mode); err != nil { - return err - } - } - return nil -} - // addMaps appends map data left to right to the template data receiver. func (td TemplateData) addMaps(maps ...map[string]string) TemplateData { for _, m := range maps { diff --git a/pkg/rocket/capcomm_test.go b/pkg/rocket/capcomm_test.go index 17c4181..6b14b5b 100644 --- a/pkg/rocket/capcomm_test.go +++ b/pkg/rocket/capcomm_test.go @@ -2,16 +2,20 @@ package rocket import ( "context" + "io" "os" "runtime" "strings" "testing" + + "github.com/nehemming/cirocket/pkg/loggee/stdlog" + "github.com/nehemming/cirocket/pkg/providers" ) const testConfigFile = "testdir/file.yml" func TestCapCommConfigFile(t *testing.T) { - capComm := newCapCommFromEnvironment(testConfigFile) + capComm := newCapCommFromEnvironment(testConfigFile, stdlog.New()) if capComm.params.Get(ConfigFileFullPath) == "" { t.Error("Base params missing ", ConfigFileFullPath) @@ -28,7 +32,7 @@ func TestCapCommConfigFile(t *testing.T) { } func TestCapCommRuntime(t *testing.T) { - capComm := newCapCommFromEnvironment(testConfigFile) + capComm := newCapCommFromEnvironment(testConfigFile, stdlog.New()) if capComm.runtime.GOARCH != runtime.GOARCH { t.Error("Unexpected GOARCH in runtime", capComm.runtime.GOARCH) @@ -43,7 +47,7 @@ func TestCapCommEnv(t *testing.T) { os.Setenv("TEST_ENV_CAPCOMM", "99") defer os.Unsetenv("TEST_ENV_CAPCOMM") - capComm := newCapCommFromEnvironment(testConfigFile) + capComm := newCapCommFromEnvironment(testConfigFile, stdlog.New()) if capComm.env == nil { t.Error("Unexpected nil env") @@ -58,8 +62,17 @@ func TestCapCommEnv(t *testing.T) { } } +func TestNewCapCommFromTestLog(t *testing.T) { + l := stdlog.New() + capComm := newCapCommFromEnvironment(testConfigFile, l) + + if capComm.Log() != l { + t.Error("log issue") + } +} + func TestNewCapCommFromEnvironment(t *testing.T) { - capComm := newCapCommFromEnvironment(testConfigFile) + capComm := newCapCommFromEnvironment(testConfigFile, stdlog.New()) if capComm.data != nil { t.Error("Unexpected non nil data") @@ -93,8 +106,32 @@ func TestNewCapCommFromEnvironment(t *testing.T) { t.Error("Unexpected non nil mission") } - if capComm.ioSettings == nil { - t.Error("Unexpected nil ioSettings") + if capComm.resources == nil { + t.Error("Unexpected nil resources") + } +} + +func TestNewCapCommFromEnvironmentResources(t *testing.T) { + capComm := newCapCommFromEnvironment(testConfigFile, stdlog.New()) + + if capComm.resources == nil { + t.Error("Unexpected nil resources") + } + + if len(capComm.resources) != 5 { + t.Error("should have 6 resources", len(capComm.resources)) + } +} + +func TestNewCapCommFromEnvironmentVariables(t *testing.T) { + capComm := newCapCommFromEnvironment(testConfigFile, stdlog.New()) + + if capComm.exportTo != nil { + t.Error("exportTo should be nil") + } + + if capComm.variables == nil { + t.Error("Unexpected nil variables") } } @@ -102,7 +139,7 @@ func TestCapCommCopyEnvTrusted(t *testing.T) { os.Setenv("TEST_ENV_CAPCOMM", "99") defer os.Unsetenv("TEST_ENV_CAPCOMM") - root := newCapCommFromEnvironment(testConfigFile) + root := newCapCommFromEnvironment(testConfigFile, stdlog.New()) capComm := root.Copy(false) if capComm.env == nil { @@ -116,13 +153,17 @@ func TestCapCommCopyEnvTrusted(t *testing.T) { if capComm.env.Get("UNKNOWN_PARAM_FROM_OS") != "" { t.Error("env returning empty string for missing os env") } + + if capComm.exportTo == nil { + t.Error("exportTo should not be nil") + } } func TestCapCommCopyEnvNotTrusted(t *testing.T) { os.Setenv("TEST_ENV_CAPCOMM", "99") defer os.Unsetenv("TEST_ENV_CAPCOMM") - root := newCapCommFromEnvironment(testConfigFile) + root := newCapCommFromEnvironment(testConfigFile, stdlog.New()) capComm := root.Copy(true) if capComm.env == nil { @@ -140,10 +181,14 @@ func TestCapCommCopyEnvNotTrusted(t *testing.T) { if capComm.entrustedParentEnv.Get("TEST_ENV_CAPCOMM") != "99" { t.Error("entrustedParentEnv not pulling from os env") } + + if capComm.exportTo == nil { + t.Error("exportTo should not be nil") + } } func TestCapCommRuntimeCopyTrusted(t *testing.T) { - root := newCapCommFromEnvironment(testConfigFile) + root := newCapCommFromEnvironment(testConfigFile, stdlog.New()) capComm := root.Copy(false) if capComm.runtime.GOARCH != runtime.GOARCH { @@ -155,8 +200,8 @@ func TestCapCommRuntimeCopyTrusted(t *testing.T) { } } -func TestCapCommRuntimeCopyTNotrusted(t *testing.T) { - root := newCapCommFromEnvironment(testConfigFile) +func TestCapCommRuntimeCopyNotTrusted(t *testing.T) { + root := newCapCommFromEnvironment(testConfigFile, stdlog.New()) capComm := root.Copy(true) if capComm.runtime.GOARCH != runtime.GOARCH { @@ -169,7 +214,7 @@ func TestCapCommRuntimeCopyTNotrusted(t *testing.T) { } func TestCapCommCopyUnSealed(t *testing.T) { - root := newCapCommFromEnvironment(testConfigFile) + root := newCapCommFromEnvironment(testConfigFile, stdlog.New()) capComm := root.Copy(false) if capComm.sealed { @@ -187,7 +232,7 @@ func TestCopyTrusted(t *testing.T) { os.Setenv("TEST_ENV_CAPCOMM", "99") defer os.Unsetenv("TEST_ENV_CAPCOMM") - root := newCapCommFromEnvironment(testConfigFile) + root := newCapCommFromEnvironment(testConfigFile, stdlog.New()) capComm := root.Copy(false) if capComm.data != nil { @@ -218,8 +263,8 @@ func TestCopyTrusted(t *testing.T) { t.Error("Unexpected non nil mission") } - if capComm.ioSettings == nil { - t.Error("Unexpected nil ioSettings") + if len(capComm.resources) != 5 { + t.Error("Unexpected resources", len(capComm.resources)) } } @@ -227,7 +272,7 @@ func TestCopyNoTrust(t *testing.T) { os.Setenv("TEST_ENV_CAPCOMM", "99") defer os.Unsetenv("TEST_ENV_CAPCOMM") - root := newCapCommFromEnvironment(testConfigFile) + root := newCapCommFromEnvironment(testConfigFile, stdlog.New()) capComm := root.Copy(true) if capComm.data != nil { @@ -258,13 +303,13 @@ func TestCopyNoTrust(t *testing.T) { t.Error("Unexpected non nil mission") } - if capComm.ioSettings == nil { - t.Error("Unexpected nil ioSettings") + if len(capComm.resources) != 5 { + t.Error("Unexpected resources", len(capComm.resources)) } } func TestWithMission(t *testing.T) { - capComm := newCapCommFromEnvironment(testConfigFile).Copy(false) + capComm := newCapCommFromEnvironment(testConfigFile, stdlog.New()).Copy(false) if capComm.mission != nil { t.Error("Unexpected non nil mission") @@ -286,7 +331,7 @@ func TestWithMission(t *testing.T) { } func TestMergeBasicEnvMap(t *testing.T) { - capComm := newCapCommFromEnvironment(testConfigFile).Copy(false) + capComm := newCapCommFromEnvironment(testConfigFile, stdlog.New()).Copy(false) envMap := make(EnvMap) @@ -314,7 +359,7 @@ func TestMergeBasicEnvMap(t *testing.T) { } func TestAddAdditionalMissionData(t *testing.T) { - capComm := newCapCommFromEnvironment(testConfigFile).Copy(false) + capComm := newCapCommFromEnvironment(testConfigFile, stdlog.New()).Copy(false) capComm.AddAdditionalMissionData(nil) @@ -352,213 +397,254 @@ func TestAddAdditionalMissionData(t *testing.T) { } } -func TestAddFile(t *testing.T) { //nolint -- keep as one test - envMap := make(EnvMap) - envMap["something"] = "here" +func TestAddFileResource(t *testing.T) { //nolint -- keep as one test ctx := context.Background() - capComm := newCapCommFromEnvironment(testConfigFile).Copy(false).MergeBasicEnvMap(envMap) + capComm := newCapCommFromEnvironment(testConfigFile, stdlog.New()).Copy(false) - if err := capComm.AddFile(ctx, OutputIO, "test123", IOModeOutput|IOModeAppend); err != nil { - t.Error("AddFile error", err) + if err := capComm.AddFileResource(ctx, OutputIO, "test123", providers.IOModeOutput|providers.IOModeAppend); err != nil { + t.Error("AddFileResource error", err) } - if f, ok := capComm.ioSettings.files[OutputIO]; !ok { - t.Error("No entry") - } else if f.filePath != "test123" { - t.Error("Output filePath wrong", f.filePath) + var outputProvider providers.ResourceProvider + if f, ok := capComm.resources[OutputIO]; !ok { + t.Error("No entry OutputIO") + } else { + outputProvider = f } - if err := capComm.AddFile(ctx, OutputIO, "test456", IOModeOutput|IOModeAppend); err != nil { - t.Error("AddFile(2) error", err) + if err := capComm.AddFileResource(ctx, OutputIO, "test456", providers.IOModeOutput|providers.IOModeAppend); err != nil { + t.Error("AddFileResource(2) error", err) } - if f, ok := capComm.ioSettings.files[OutputIO]; !ok { - t.Error("No entry") - } else if f.FilePath() != "test456" { - t.Error("Output filePath wrong (2)", f.filePath) + if f, ok := capComm.resources[OutputIO]; !ok { + t.Error("No entry OutputIO (2)") + } else if outputProvider == f { + t.Error("Resource not replaced") } +} + +func TestAddFileResourceExpansion(t *testing.T) { + envMap := make(EnvMap) + envMap["something"] = "config" + ctx := context.Background() + capComm := newCapCommFromEnvironment(testConfigFile, stdlog.New()).Copy(false).MergeBasicEnvMap(envMap) - if err := capComm.AddFile(ctx, OutputIO, "test${something}", IOModeOutput|IOModeAppend); err != nil { - t.Error("AddFile(3) error", err) + if err := capComm.AddFileResource(ctx, InputIO, "${something}.go", providers.IOModeInput); err != nil { + t.Error("AddFileResource error", err) } - if f, ok := capComm.ioSettings.files[OutputIO]; !ok { + if f, ok := capComm.resources[InputIO]; !ok { t.Error("No entry") - } else if f.filePath != "testhere" { - t.Error("Output filePath wrong (3)", f.filePath) - } + } else { + r, err := f.OpenRead(ctx) + if err != nil { + t.Error("OpenRead error, missing file expansion?", err) + return + } - if err := capComm.AddFile(ctx, OutputIO, "test--{{.Env.something}}", IOModeOutput|IOModeAppend); err != nil { - t.Error("AddFile(4) error", err) - } + defer r.Close() - if f, ok := capComm.ioSettings.files[OutputIO]; !ok { - t.Error("No entry") - } else if f.filePath != "test--here" { - t.Error("Output filePath wrong (4)", f.filePath) - } + b, err := io.ReadAll(r) - // Check GetFileDetails - fd := capComm.GetFileDetails(OutputIO) - if fd == nil { - t.Error("No fd entry") + if b == nil || err != nil { + t.Error("OpenRead error, data?", err) + } } - if !fd.InMode(IOModeOutput | IOModeAppend) { - t.Error("No fd file mode wrong", fd.fileMode) +} + +func TestGetResource(t *testing.T) { + capComm := newCapCommFromEnvironment(testConfigFile, stdlog.New()).Copy(false) + + for _, r := range []providers.ResourceID{ + OutputIO, ErrorIO, Stdin, Stdout, Stderr, + } { + res := capComm.GetResource(r) + if res == nil { + t.Error("No entry", r) + } } - if fd.filePath != "test--here" { - t.Error("Output fd wrong", fd.filePath) + + if inRes := capComm.GetResource(InputIO); inRes != nil { + t.Error("Input defined") } } -func TestAttachOutputCreateMode(t *testing.T) { +func TestAttachOutputSpecFileCreateMode(t *testing.T) { ctx := context.Background() - capComm := newCapCommFromEnvironment(testConfigFile).Copy(false) + capComm := newCapCommFromEnvironment(testConfigFile, stdlog.New()).Copy(false) outSpec := OutputSpec{ - Output: "test1234", + Path: "test1234", } - if err := capComm.AttachOutput(ctx, outSpec); err != nil { + if err := capComm.AttachOutputSpec(ctx, OutputIO, outSpec); err != nil { t.Error("AttachOutput error", err) } - fd := capComm.GetFileDetails(OutputIO) - if fd == nil { - t.Error("No fd entry") - } - if !fd.InMode(IOModeOutput | IOModeTruncate) { - t.Error("No fd file mode wrong", fd.fileMode) + res := capComm.GetResource(OutputIO) + if res == nil { + t.Error("No res entry") } - if fd.filePath != "test1234" { - t.Error("Output fd wrong", fd.filePath) + + if fd, ok := res.(providers.FileDetail); !ok { + t.Error("No res is not a file") + } else { + if !fd.InMode(providers.IOModeOutput | providers.IOModeTruncate) { + t.Error("No fd file mode wrong", fd.IOMode()) + } + if fd.FilePath() != "test1234" { + t.Error("Output fd wrong", fd.FilePath()) + } } } -func TestAttachOutputAppendMode(t *testing.T) { +func TestAttachOutputSpecFileAppendMode(t *testing.T) { ctx := context.Background() - capComm := newCapCommFromEnvironment(testConfigFile).Copy(false) + capComm := newCapCommFromEnvironment(testConfigFile, stdlog.New()).Copy(false) outSpec := OutputSpec{ - Output: "test1234", - AppendOutput: true, + Path: "test1234", + Append: true, } - if err := capComm.AttachOutput(ctx, outSpec); err != nil { + if err := capComm.AttachOutputSpec(ctx, OutputIO, outSpec); err != nil { t.Error("AttachOutput error", err) } - fd := capComm.GetFileDetails(OutputIO) - if fd == nil { - t.Error("No fd entry") + res := capComm.GetResource(OutputIO) + if res == nil { + t.Error("No res entry") + return } - if !fd.InMode(IOModeOutput | IOModeAppend) { - t.Error("No fd file mode wrong", fd.fileMode) + + fd := res.(providers.FileDetail) + + if !fd.InMode(providers.IOModeOutput | providers.IOModeAppend) { + t.Error("No fd io mode wrong", fd.IOMode()) } - if fd.filePath != "test1234" { - t.Error("Output fd wrong", fd.filePath) + if fd.FilePath() != "test1234" { + t.Error("Output fd wrong", fd.FilePath()) } } func TestAttachRedirectNoMergeStdOut(t *testing.T) { ctx := context.Background() - capComm := newCapCommFromEnvironment(testConfigFile).Copy(false) + capComm := newCapCommFromEnvironment(testConfigFile, stdlog.New()).Copy(false) redirect := Redirection{ - OutputSpec: OutputSpec{ - Output: "test1234", - AppendOutput: true, + Output: &OutputSpec{ + Path: "test1234", + Append: true, + }, + Input: &InputSpec{ + Path: "in2020", + }, + Error: &OutputSpec{ + Path: "sometimes", }, - Input: "in2020", - Error: "sometimes", } if err := capComm.AttachRedirect(ctx, redirect); err != nil { t.Error("AttachRedirect error", err) } - fdOut := capComm.GetFileDetails(OutputIO) - if fdOut == nil { - t.Error("No fdOut entry") + resOut := capComm.GetResource(OutputIO) + if resOut == nil { + t.Error("No resOut") } - if !fdOut.InMode(IOModeOutput | IOModeAppend) { - t.Error("No fdOut file mode wrong", fdOut.fileMode) + + fdOut := resOut.(providers.FileDetail) + if !fdOut.InMode(providers.IOModeOutput | providers.IOModeAppend) { + t.Error("No fdOut io mode wrong", fdOut.IOMode()) } - if fdOut.filePath != "test1234" { - t.Error("Output fdOut wrong", fdOut.filePath) + if fdOut.FilePath() != "test1234" { + t.Error("Output fdOut wrong", fdOut.FilePath()) } } func TestAttachRedirectNoMergeStdErr(t *testing.T) { ctx := context.Background() - capComm := newCapCommFromEnvironment(testConfigFile).Copy(false) + capComm := newCapCommFromEnvironment(testConfigFile, stdlog.New()).Copy(false) redirect := Redirection{ - OutputSpec: OutputSpec{ - Output: "test1234", - AppendOutput: true, + Output: &OutputSpec{ + Path: "test1234", + Append: true, + }, + Input: &InputSpec{ + Path: "in2020", + }, + Error: &OutputSpec{ + Path: "sometimes", }, - Input: "in2020", - Error: "sometimes", } if err := capComm.AttachRedirect(ctx, redirect); err != nil { t.Error("AttachRedirect error", err) } - fdError := capComm.GetFileDetails(ErrorIO) - if fdError == nil { - t.Error("No fdError entry") + resErr := capComm.GetResource(ErrorIO) + if resErr == nil { + t.Error("No resErr") } - if !fdError.InMode(IOModeError | IOModeTruncate) { - t.Error("No fdError file mode wrong", fdError.fileMode) + + fdError := resErr.(providers.FileDetail) + if !fdError.InMode(providers.IOModeError | providers.IOModeTruncate) { + t.Error("No fdError io mode wrong", fdError.IOMode()) } - if fdError.filePath != "sometimes" { - t.Error("Output fdError wrong", fdError.filePath) + if fdError.FilePath() != "sometimes" { + t.Error("Error path fdError wrong", fdError.FilePath()) } } func TestAttachRedirectNoMergeStdIn(t *testing.T) { ctx := context.Background() - capComm := newCapCommFromEnvironment(testConfigFile).Copy(false) + capComm := newCapCommFromEnvironment(testConfigFile, stdlog.New()).Copy(false) redirect := Redirection{ - OutputSpec: OutputSpec{ - Output: "test1234", - AppendOutput: true, + Output: &OutputSpec{ + Path: "test1234", + Append: true, + }, + Input: &InputSpec{ + Path: "in2020", + }, + Error: &OutputSpec{ + Path: "sometimes", }, - Input: "in2020", - Error: "sometimes", } if err := capComm.AttachRedirect(ctx, redirect); err != nil { t.Error("AttachRedirect error", err) } - fdIn := capComm.GetFileDetails(InputIO) - if fdIn == nil { - t.Error("No fdIn entry") + resIn := capComm.GetResource(InputIO) + if resIn == nil { + t.Error("No resIn") } - if !fdIn.InMode(IOModeInput) { - t.Error("No fdIn file mode wrong", fdIn.fileMode) + + fdIn := resIn.(providers.FileDetail) + if !fdIn.InMode(providers.IOModeInput) { + t.Error("No fdIn io mode wrong", fdIn.IOMode()) } - if fdIn.filePath != "in2020" { - t.Error("Output fdIn wrong", fdIn.filePath) + if fdIn.FilePath() != "in2020" { + t.Error("Error path fdIn wrong", fdIn.FilePath()) } } func TestAttachRedirectErrorMergeStdOut(t *testing.T) { ctx := context.Background() - capComm := newCapCommFromEnvironment(testConfigFile).Copy(false) + capComm := newCapCommFromEnvironment(testConfigFile, stdlog.New()).Copy(false) redirect := Redirection{ - OutputSpec: OutputSpec{ - Output: "test1234", - AppendOutput: true, + Output: &OutputSpec{ + Path: "test1234", + Append: true, + }, + Input: &InputSpec{ + Path: "in2020", }, - Input: "in2020", - Error: "sometimes", MergeErrorWithOutput: true, } @@ -566,79 +652,81 @@ func TestAttachRedirectErrorMergeStdOut(t *testing.T) { t.Error("AttachRedirect error", err) } - fdOut := capComm.GetFileDetails(OutputIO) - if fdOut == nil { - t.Error("No fdOut entry") + resOut := capComm.GetResource(OutputIO) + if resOut == nil { + t.Error("No resOut") } - if !fdOut.InMode(IOModeOutput | IOModeError | IOModeAppend) { - t.Error("No fdOut file mode wrong", fdOut.fileMode) + fdOut := resOut.(providers.FileDetail) + if !fdOut.InMode(providers.IOModeOutput | providers.IOModeError | providers.IOModeAppend) { + t.Error("No fdOut error io mode wrong", fdOut.IOMode()) } - if fdOut.filePath != "test1234" { - t.Error("Output fdOut wrong", fdOut.filePath) + + resErr := capComm.GetResource(ErrorIO) + if resErr != resOut { + t.Error("resErr different from resOut") } } func TestAttachRedirectErrorMergeStdErr(t *testing.T) { ctx := context.Background() - capComm := newCapCommFromEnvironment(testConfigFile).Copy(false) + capComm := newCapCommFromEnvironment(testConfigFile, stdlog.New()).Copy(false) redirect := Redirection{ - OutputSpec: OutputSpec{ - Output: "test1234", - AppendOutput: true, - }, - Input: "in2020", - Error: "sometimes", - MergeErrorWithOutput: true, + DirectError: true, } if err := capComm.AttachRedirect(ctx, redirect); err != nil { t.Error("AttachRedirect error", err) } - fdError := capComm.GetFileDetails(ErrorIO) - if fdError == nil { - t.Error("No fdError entry") + resErr := capComm.GetResource(ErrorIO) + if resErr == nil { + t.Error("No resErr") } - if !fdError.InMode(IOModeOutput | IOModeError | IOModeAppend) { - t.Error("No fdError file mode wrong", fdError.fileMode) + resStdErr := capComm.GetResource(Stderr) + if resStdErr == nil { + t.Error("No resStdErr") } - if fdError.filePath != "test1234" { - t.Error("Output fdError wrong", fdError.filePath) + + if resErr != resStdErr { + t.Error("resErr different from resStdErr") } } -func TestAttachRedirectErrorMergeStdIn(t *testing.T) { +func TestAttachRedirectOutputExpand(t *testing.T) { + os.Setenv("TEST_ENV_CAPCOMM", "99") + defer os.Unsetenv("TEST_ENV_CAPCOMM") + + envMap := make(EnvMap) + envMap["something"] = "here" + ctx := context.Background() - capComm := newCapCommFromEnvironment(testConfigFile).Copy(false) + capComm := newCapCommFromEnvironment(testConfigFile, stdlog.New()).Copy(false).MergeBasicEnvMap(envMap) redirect := Redirection{ - OutputSpec: OutputSpec{ - Output: "test1234", - AppendOutput: true, + Output: &OutputSpec{ + Path: "{{.Env.something}}-test1234", }, - Input: "in2020", - Error: "sometimes", - MergeErrorWithOutput: true, } if err := capComm.AttachRedirect(ctx, redirect); err != nil { t.Error("AttachRedirect error", err) } - fdIn := capComm.GetFileDetails(InputIO) - if fdIn == nil { - t.Error("No fdIn entry") + resOut := capComm.GetResource(OutputIO) + if resOut == nil { + t.Error("No resOut") } - if !fdIn.InMode(IOModeInput) { - t.Error("No fdIn file mode wrong", fdIn.fileMode) + fdOut := resOut.(providers.FileDetail) + if !fdOut.InMode(providers.IOModeOutput | providers.IOModeTruncate) { + t.Error("No fdOut io mode wrong", fdOut.IOMode()) } - if fdIn.filePath != "in2020" { - t.Error("Output fdIn wrong", fdIn.filePath) + if fdOut.FilePath() != "here-test1234" { + t.Error("Output fdOut wrong", fdOut.FilePath()) } } -func TestAttachRedirectOutputExpand(t *testing.T) { +func TestAttachRedirectErrorExpand(t *testing.T) { os.Setenv("TEST_ENV_CAPCOMM", "99") defer os.Unsetenv("TEST_ENV_CAPCOMM") @@ -646,103 +734,106 @@ func TestAttachRedirectOutputExpand(t *testing.T) { envMap["something"] = "here" ctx := context.Background() - capComm := newCapCommFromEnvironment(testConfigFile).Copy(false).MergeBasicEnvMap(envMap) + capComm := newCapCommFromEnvironment(testConfigFile, stdlog.New()).Copy(false).MergeBasicEnvMap(envMap) redirect := Redirection{ - OutputSpec: OutputSpec{ - Output: "{{.Env.something}}-test1234", - AppendOutput: false, + Error: &OutputSpec{ + Path: "sometimes${TEST_ENV_CAPCOMM}", + Append: true, }, - Input: "in2020", - Error: "sometimes${TEST_ENV_CAPCOMM}", - AppendError: true, } if err := capComm.AttachRedirect(ctx, redirect); err != nil { t.Error("AttachRedirect error", err) } - fdOut := capComm.GetFileDetails(OutputIO) - if fdOut == nil { - t.Error("No fdOut entry") + resErr := capComm.GetResource(ErrorIO) + if resErr == nil { + t.Error("No resErr") } - if !fdOut.InMode(IOModeOutput | IOModeTruncate) { - t.Error("No fdOut file mode wrong", fdOut.fileMode) + + fdError := resErr.(providers.FileDetail) + if !fdError.InMode(providers.IOModeError | providers.IOModeAppend) { + t.Error("No fdError io mode wrong", fdError.IOMode()) } - if fdOut.filePath != "here-test1234" { - t.Error("Output fdOut wrong", fdOut.filePath) + if fdError.FilePath() != "sometimes99" { + t.Error("Output fdError wrong", fdError.FilePath()) } } -func TestAttachRedirectErrorExpand(t *testing.T) { +func TestAttachRedirectInExpand(t *testing.T) { os.Setenv("TEST_ENV_CAPCOMM", "99") defer os.Unsetenv("TEST_ENV_CAPCOMM") - envMap := make(EnvMap) - envMap["something"] = "here" - ctx := context.Background() - capComm := newCapCommFromEnvironment(testConfigFile).Copy(false).MergeBasicEnvMap(envMap) + capComm := newCapCommFromEnvironment(testConfigFile, stdlog.New()).Copy(false) redirect := Redirection{ - OutputSpec: OutputSpec{ - Output: "{{.Env.something}}-test1234", - AppendOutput: false, + Input: &InputSpec{ + Inline: "{{.Env.TEST_ENV_CAPCOMM}}-test1234", }, - Input: "in2020", - Error: "sometimes${TEST_ENV_CAPCOMM}", - AppendError: true, } if err := capComm.AttachRedirect(ctx, redirect); err != nil { t.Error("AttachRedirect error", err) } - fdError := capComm.GetFileDetails(ErrorIO) - if fdError == nil { - t.Error("No fdError entry") + resIn := capComm.GetResource(InputIO) + if resIn == nil { + t.Error("No resIn") } - if !fdError.InMode(IOModeError | IOModeAppend) { - t.Error("No fdError file mode wrong", fdError.fileMode) + + r, err := resIn.OpenRead(ctx) + if err != nil { + t.Error("open inline", err) } - if fdError.filePath != "sometimes99" { - t.Error("Output fdError wrong", fdError.filePath) + + b, err := io.ReadAll(r) + if err != nil { + t.Error("read inline", err) + return } -} -func TestAttachRedirectInExpand(t *testing.T) { - os.Setenv("TEST_ENV_CAPCOMM", "99") - defer os.Unsetenv("TEST_ENV_CAPCOMM") + if string(b) != "99-test1234" { + t.Error("string mismatch", string(b)) + } +} +func TestAttachRedirectInExpandNotSet(t *testing.T) { envMap := make(EnvMap) envMap["something"] = "here" ctx := context.Background() - capComm := newCapCommFromEnvironment(testConfigFile).Copy(false).MergeBasicEnvMap(envMap) + capComm := newCapCommFromEnvironment(testConfigFile, stdlog.New()).Copy(false).MergeBasicEnvMap(envMap) redirect := Redirection{ - OutputSpec: OutputSpec{ - Output: "{{.Env.something}}-test1234", - AppendOutput: false, + Input: &InputSpec{ + Inline: "{{.Env.TEST_ENV_CAPCOMM}}-test1234", }, - Input: "in2020", - Error: "sometimes${TEST_ENV_CAPCOMM}", - AppendError: true, } if err := capComm.AttachRedirect(ctx, redirect); err != nil { t.Error("AttachRedirect error", err) } - fdIn := capComm.GetFileDetails(InputIO) - if fdIn == nil { - t.Error("No fdIn entry") + resIn := capComm.GetResource(InputIO) + if resIn == nil { + t.Error("No resIn") } - if !fdIn.InMode(IOModeInput) { - t.Error("No fdIn file mode wrong", fdIn.fileMode) + + r, err := resIn.OpenRead(ctx) + if err != nil { + t.Error("open inline", err) + } + + b, err := io.ReadAll(r) + if err != nil { + t.Error("read inline", err) + return } - if fdIn.filePath != "in2020" { - t.Error("Output fdIn wrong", fdIn.filePath) + + if string(b) != "-test1234" { + t.Error("string mismatch", string(b)) } } @@ -751,7 +842,7 @@ func TestMergeParams(t *testing.T) { defer os.Unsetenv("TEST_ENV_CAPCOMM") ctx := context.Background() - capComm := newCapCommFromEnvironment(testConfigFile).Copy(false) + capComm := newCapCommFromEnvironment(testConfigFile, stdlog.New()).Copy(false) if err := capComm.MergeParams(ctx, nil); err != nil { t.Error("nil MergeParams error", err) @@ -793,12 +884,12 @@ func TestMergeParamsWithFile(t *testing.T) { envMap["name"] = "config" ctx := context.Background() - capComm := newCapCommFromEnvironment(testConfigFile).Copy(false).MergeBasicEnvMap(envMap) + capComm := newCapCommFromEnvironment(testConfigFile, stdlog.New()).Copy(false).MergeBasicEnvMap(envMap) params := make([]Param, 0) params = append(params, Param{ Name: "test", - File: "{{.Env.name}}.go", + Path: "{{.Env.name}}.go", }) if err := capComm.MergeParams(ctx, params); err != nil { @@ -816,13 +907,13 @@ func TestMergeParamsWithOptionalFile(t *testing.T) { envMap["name"] = "notaconfig" ctx := context.Background() - capComm := newCapCommFromEnvironment(testConfigFile).Copy(false).MergeBasicEnvMap(envMap) + capComm := newCapCommFromEnvironment(testConfigFile, stdlog.New()).Copy(false).MergeBasicEnvMap(envMap) params := make([]Param, 0) params = append(params, Param{ - Name: "test", - File: "{{.Env.name}}.go", - FileOptional: true, + Name: "test", + Path: "{{.Env.name}}.go", + Optional: true, }) if err := capComm.MergeParams(ctx, params); err != nil { @@ -840,13 +931,13 @@ func TestMergeParamsWithSkipTemplate(t *testing.T) { envMap["name"] = "notaconfig" ctx := context.Background() - capComm := newCapCommFromEnvironment(testConfigFile).Copy(false).MergeBasicEnvMap(envMap) + capComm := newCapCommFromEnvironment(testConfigFile, stdlog.New()).Copy(false).MergeBasicEnvMap(envMap) params := make([]Param, 0) params = append(params, Param{ - Name: "test", - Value: "{{.Env.name}}.go", - SkipTemplate: true, + Name: "test", + Value: "{{.Env.name}}.go", + SkipExpand: true, }) if err := capComm.MergeParams(ctx, params); err != nil { @@ -864,7 +955,7 @@ func TestMergeParamsWithParam(t *testing.T) { envMap["name"] = "config" ctx := context.Background() - capComm := newCapCommFromEnvironment(testConfigFile).Copy(false).MergeBasicEnvMap(envMap) + capComm := newCapCommFromEnvironment(testConfigFile, stdlog.New()).Copy(false).MergeBasicEnvMap(envMap) params := make([]Param, 0) params = append(params, Param{ @@ -903,7 +994,7 @@ func TestMergeTemplateEnvs(t *testing.T) { defer os.Unsetenv("TEST_ENV_CAPCOMM") ctx := context.Background() - capComm := newCapCommFromEnvironment(testConfigFile).Copy(false) + capComm := newCapCommFromEnvironment(testConfigFile, stdlog.New()).Copy(false) envMap := make(EnvMap) @@ -925,7 +1016,7 @@ func TestMergeTemplateEnvs(t *testing.T) { } func TestFuncMap(t *testing.T) { - capComm := newCapCommFromEnvironment(testConfigFile).Copy(false) + capComm := newCapCommFromEnvironment(testConfigFile, stdlog.New()).Copy(false) fm := capComm.FuncMap() if len(fm) != 1 { @@ -937,7 +1028,7 @@ func TestFGetExecEnvNoOSInherit(t *testing.T) { os.Setenv("TEST_ENV_CAPCOMM", "99") defer os.Unsetenv("TEST_ENV_CAPCOMM") - execEnv := newCapCommFromEnvironment(testConfigFile).Copy(true).GetExecEnv() + execEnv := newCapCommFromEnvironment(testConfigFile, stdlog.New()).Copy(true).GetExecEnv() if len(execEnv) != 0 { t.Error("execEnv len not 0", len(execEnv)) @@ -947,7 +1038,7 @@ func TestFGetExecEnvNoOSInherit(t *testing.T) { func TestGetExecEnv(t *testing.T) { envMap := make(EnvMap) envMap["TEST_ENV_CAPCOMM"] = "99" - execEnv := newCapCommFromEnvironment(testConfigFile).Copy(true).MergeBasicEnvMap(envMap).GetExecEnv() + execEnv := newCapCommFromEnvironment(testConfigFile, stdlog.New()).Copy(true).MergeBasicEnvMap(envMap).GetExecEnv() if len(execEnv) != 1 { t.Error("execEnv len not 1", len(execEnv)) @@ -957,7 +1048,7 @@ func TestGetExecEnv(t *testing.T) { } func TestIsFilteredInclude(t *testing.T) { - capComm := newCapCommFromEnvironment(testConfigFile).Copy(true) + capComm := newCapCommFromEnvironment(testConfigFile, stdlog.New()).Copy(true) if capComm.isFiltered(nil) != false { t.Error("Nil should not filter") @@ -979,7 +1070,7 @@ func TestIsFilteredInclude(t *testing.T) { } func TestIsFilteredNotInclude(t *testing.T) { - capComm := newCapCommFromEnvironment(testConfigFile).Copy(true) + capComm := newCapCommFromEnvironment(testConfigFile, stdlog.New()).Copy(true) if capComm.isFiltered(nil) != false { t.Error("Nil should not filter") @@ -998,7 +1089,7 @@ func TestIsFilteredNotInclude(t *testing.T) { } func TestIsFilteredExclude(t *testing.T) { - capComm := newCapCommFromEnvironment(testConfigFile).Copy(true) + capComm := newCapCommFromEnvironment(testConfigFile, stdlog.New()).Copy(true) if capComm.isFiltered(nil) != false { t.Error("Nil should not filter") @@ -1018,7 +1109,7 @@ func TestIsFilteredExclude(t *testing.T) { } func TestIsFilteredNotExclude(t *testing.T) { - capComm := newCapCommFromEnvironment(testConfigFile).Copy(true) + capComm := newCapCommFromEnvironment(testConfigFile, stdlog.New()).Copy(true) if capComm.isFiltered(nil) != false { t.Error("Nil should not filter") @@ -1044,5 +1135,358 @@ func TestMustNotBeSealed(t *testing.T) { } }() - newCapCommFromEnvironment(testConfigFile).mustNotBeSealed() + newCapCommFromEnvironment(testConfigFile, stdlog.New()).mustNotBeSealed() +} + +func TestIndentTemplateFunc(t *testing.T) { + ctx := context.Background() + capComm := newCapCommFromEnvironment(testConfigFile, stdlog.New()).Copy(true) + + s, err := capComm.ExpandString(ctx, "spacing", `{{"\nhello"|Indent 6}} + `) + if err != nil { + t.Error("unexpected error", err) + } + + if !strings.HasPrefix(s, "\n hello") { + t.Error("indent missing", s) + } +} + +func TestIndentFirstLineTemplateFunc(t *testing.T) { + ctx := context.Background() + capComm := newCapCommFromEnvironment(testConfigFile, stdlog.New()).Copy(true) + + s, err := capComm.ExpandString(ctx, "spacing", `{{"hello"|Indent 6}} + `) + if err != nil { + t.Error("unexpected error", err) + } + + if !strings.HasPrefix(s, "hello") { + t.Error("indent present", s) + } +} + +func TestExportVariable(t *testing.T) { + capComm := newCapCommFromEnvironment(testConfigFile, stdlog.New()).Copy(true) + + if len(capComm.variables) != 0 { + t.Error("pre existing vars", len(capComm.variables)) + } + + taskCapCom := capComm.Copy(true) + + taskCapCom.ExportVariable("hello", "there") + + if v, ok := capComm.variables["hello"]; !ok || v != "there" { + t.Error("var not exported", ok, v) + } +} + +func TestValidateInputSpecEmpty(t *testing.T) { + inputSpec := &InputSpec{} + + if err := validateInputSpec(inputSpec); err == nil || err.Error() != "no input source was specified" { + t.Error("input spec empty check fails") + } +} + +func TestValidateInputSpecMultiple(t *testing.T) { + for i, r := range []InputSpec{ + { + Inline: "-", + Path: "-", + }, + { + Inline: "-", + Variable: "-", + }, + { + Inline: "-", + URL: "-", + }, + { + Path: "-", + Variable: "-", + }, + { + Path: "-", + URL: "-", + }, + { + Variable: "-", + URL: "-", + }, + } { + if err := validateInputSpec(&r); err == nil || err.Error() != "more than one input source was specified, only one is permitted" { + t.Error("input spec multi check fails", i, r) + } + } +} + +func TestCreateProviderFromInputSpec(t *testing.T) { + ctx := context.Background() + capComm := newCapCommFromEnvironment(testConfigFile, stdlog.New()).Copy(true) + + capComm.ExportVariable("test_it", "hello") + + for i, r := range []InputSpec{ + { + Inline: "12345", + }, + { + Path: "testdata/six.yml", + }, + { + URL: "https://raw.githubusercontent.com/nehemming/cirocket/master/README.md", + }, + { + Variable: "test_it", + }, + } { + rp, err := capComm.createProviderFromInputSpec(ctx, r) + if err != nil { + t.Error("unexpected error", i, err) + } + + _, err = rp.OpenWrite(ctx) + if err == nil { + t.Error("open write", i) + return + } + + r, err := rp.OpenRead(ctx) + if err != nil { + t.Error("error open read", i, err) + return + } + r.Close() + } +} + +func TestAttachInputSpec(t *testing.T) { + ctx := context.Background() + capComm := newCapCommFromEnvironment(testConfigFile, stdlog.New()).Copy(true) + + capComm.ExportVariable("test_it", "hello") + + var rp providers.ResourceProvider + for i, r := range []InputSpec{ + { + Inline: "12345", + SkipExpand: true, + }, + { + Inline: "12345", + }, + { + Path: "testdata/six.yml", + }, + { + Path: "testdata/six.yml", + SkipExpand: true, + }, + { + URL: "https://raw.githubusercontent.com/nehemming/cirocket/master/README.md", + }, + { + Variable: "test_it", + }, + } { + if err := capComm.AttachInputSpec(ctx, "test", r); err != nil { + t.Error("unexpected error", i, err) + } + + rpNext := capComm.GetResource("test") + + if rpNext == rp { + t.Error("resource update issue", i) + } + + rp = rpNext + } +} + +func TestValidateOutputSpecEmpty(t *testing.T) { + outputSpec := &OutputSpec{} + + if err := validateOutputSpec(outputSpec); err == nil || err.Error() != "no output source was specified" { + t.Error("output spec empty check fails") + } +} + +func TestValidateOutputSpecMultiple(t *testing.T) { + for i, r := range []OutputSpec{ + { + Path: "-", + Variable: "-", + }, + } { + if err := validateOutputSpec(&r); err == nil || err.Error() != "more than one output source was specified, only one is permitted" { + t.Error("output spec multi check fails", i, r) + } + } +} + +func TestCreateProviderFromoutputSpec(t *testing.T) { + ctx := context.Background() + capComm := newCapCommFromEnvironment(testConfigFile, stdlog.New()).Copy(true) + capComm.ExportVariable("test_it", "hello") + + defer func() { + _ = os.Remove("testdata/dummy.tmp") + }() + + for i, r := range []OutputSpec{ + { + Path: "testdata/dummy.tmp", + }, + { + Path: "testdata/dummy.tmp", + SkipExpand: true, + }, + { + Variable: "test_it", + }, + } { + rp, err := capComm.createProviderFromOutputSpec(ctx, r, providers.IOModeOutput) + if err != nil { + t.Error("unexpected error", i, err) + } + _, err = rp.OpenRead(ctx) + if err == nil { + t.Error("open read", i) + return + } + + w, err := rp.OpenWrite(ctx) + if err != nil { + t.Error("error open write", i, err) + return + } + + w.Close() + } + + if capComm.exportTo["test_it"] != "" { + t.Error("Variable not set") + } +} + +func TestValidateRedirection(t *testing.T) { + for i, r := range []Redirection{ + { + LogOutput: true, + Output: &OutputSpec{ + Path: "testdata/dummy.tmp", + }, + }, + { + DirectError: true, + Error: &OutputSpec{ + Path: "testdata/dummy.tmp", + }, + }, + { + MergeErrorWithOutput: true, + Error: &OutputSpec{ + Path: "testdata/dummy.tmp", + }, + }, + { + Error: &OutputSpec{}, + }, + { + Output: &OutputSpec{}, + }, + { + Input: &InputSpec{}, + }, + } { + err := validateRedirection(&r) + if err == nil { + t.Error("should fail validation", i) + return + } + } +} + +func TestGetParamFromURLSuccess(t *testing.T) { + url := "https://raw.githubusercontent.com/nehemming/cirocket/master/CREDITS" + + data, err := getParamFromURL(context.Background(), url, false) + if err != nil { + t.Error("unexpected error", err) + } + + if len(data) == 0 { + t.Error("no data") + } +} + +func TestGetParamFromURLMissingError(t *testing.T) { + url := "https://raw.githubusercontent.com/nehemming/cirocket/master/notknown" + + _, err := getParamFromURL(context.Background(), url, false) + if err == nil { + t.Error("expected error") + } +} + +func TestGetParamFromURLOptionalSuccess(t *testing.T) { + url := "https://raw.githubusercontent.com/nehemming/cirocket/master/notknown" + + data, err := getParamFromURL(context.Background(), url, true) + if err != nil { + t.Error("unexpected error", err) + } + + if len(data) != 0 { + t.Error("data") + } +} + +func TestExpandParam(t *testing.T) { + ctx := context.Background() + capComm := newCapCommFromEnvironment(testConfigFile, stdlog.New()).Copy(true) + capComm.ExportVariable("test_it", "hello") + + for i, r := range []Param{ + { + Name: "fileTest", + Path: "testdata/six.yml", + }, + { + Name: "valueTest", + Value: "1234", + }, + { + Name: "valueTest", + URL: "https://raw.githubusercontent.com/nehemming/cirocket/master/CREDITS", + }, + } { + v, err := capComm.expandParam(ctx, r) + if err != nil { + t.Error("unexpected", i, r.Name, err) + } + + if len(v) == 0 { + t.Error("zero data", i, r.Name) + } + } +} + +func TestAttachRedirectLogOutput(t *testing.T) { + ctx := context.Background() + capComm := newCapCommFromEnvironment(testConfigFile, stdlog.New()).Copy(false) + + redirect := Redirection{ + LogOutput: true, + MergeErrorWithOutput: true, + } + + if err := capComm.AttachRedirect(ctx, redirect); err != nil { + t.Error("AttachRedirect error", err) + } } diff --git a/pkg/rocket/config.go b/pkg/rocket/config.go index c40e3ae..eaed993 100644 --- a/pkg/rocket/config.go +++ b/pkg/rocket/config.go @@ -58,16 +58,21 @@ type ( // Value is the value of the parameter and is subject to expandion Value string `mapstructure:"value"` - // File is a path to file containing the value. - // If both File and Value are supplied the file will bee appended to the Valued - // The combined value will undergo template expansion - File string `mapstructure:"file"` + // Path is a path to file containing the value. + // If both Path and Value are supplied the file will bee appended to the Valued + // The combined value will undergo template expansion if SkipTemplate is false. + Path string `mapstructure:"path"` + + // URL specifies the data should come from the response body or a web request. + // The url body will be concatenated with the value and file values respectively. + // The combined value will undergo template expansion if SkipTemplate is false. + URL string `mapstructure:"url"` - // SkipTemplate skip templating the param - SkipTemplate bool `mapstructure:"skipTemplate"` + // SkipExpand skip templating the param + SkipExpand bool `mapstructure:"skipExpand"` - // FileOptional if truew allows the file not to exist - FileOptional bool `mapstructure:"optional"` + // Optional if true allows the file not to exist + Optional bool `mapstructure:"optional"` } // EnvMap is a map of environment variables to their values. @@ -169,30 +174,67 @@ type ( } OutputSpec struct { + // Variable is an exported variable available to later tasks in the same stage + Variable string `mapstructure:"variable"` + // Output is a path to a file replacing STDOUT - Output string `mapstructure:"output"` + Path string `mapstructure:"path"` // AppendOutput specifies if output should append - AppendOutput bool `mapstructure:"appendOutput"` + Append bool `mapstructure:"append"` + + // SkipExpand when true skips template expansion of the spec. + SkipExpand bool `mapstructure:"skipExpand"` + + // OS File permissions + FileMode uint `mapstructure:"fileMode"` + } + + InputSpec struct { + // Variable name to import from + Variable string `mapstructure:"variable"` + + Inline string `mapstructure:"inline"` + + // Path provides the path data. + Path string `mapstructure:"path"` + + // URl provides the data. + URL string `mapstructure:"url"` + + // Optional is true if resource can be missing. + Optional bool `mapstructure:"optional"` + + // URLTimeout request timeout, default is 30 seconds. + URLTimeout uint `mapstructure:"timeout"` + + // SkipExpand when true skips template expansion of the spec. + SkipExpand bool `mapstructure:"skipExpand"` } // Redirection is provided to a task to interpret // Redirection strings need to be expanded by the task. Redirection struct { - OutputSpec `mapstructure:",squash"` - - // Input is a file path to an existing input file replacing STDIN - Input string `mapstructure:"input"` + // Input specification + Input *InputSpec `mapstructure:"input"` - // Error is a path to a file replacing STDERR - Error string `mapstructure:"error"` + // Output specification + Output *OutputSpec `mapstructure:"output"` - // AppendError specifies if error output should append - AppendError bool `mapstructure:"appendError"` + // Error specification + Error *OutputSpec `mapstructure:"error"` // MergeErrorWithOutput specifies if error output should go to outputt // if specified Error and AppendError are ignored MergeErrorWithOutput bool `mapstructure:"merge"` + + // LogOutput if true will cause output to be logged rather than going to go to std output. + // If an output file is specified it will be used instead. + LogOutput bool `mapstructure:"logStdOut"` + + // DirectError when true causes the commands std error output to go direct to running processes std error + // When DirectError is false std error output is logged. + DirectError bool `mapstructure:"directStdErr"` } // Delims are the delimiters to use to escape template functions. diff --git a/pkg/rocket/context_test.go b/pkg/rocket/context_test.go index c81e016..23ded76 100644 --- a/pkg/rocket/context_test.go +++ b/pkg/rocket/context_test.go @@ -3,10 +3,12 @@ package rocket import ( "context" "testing" + + "github.com/nehemming/cirocket/pkg/loggee/stdlog" ) func TestContextRoundTrip(t *testing.T) { - capComm := newCapCommFromEnvironment("dir/file") + capComm := newCapCommFromEnvironment("dir/file", stdlog.New()) ctx := NewContextWithCapComm(context.Background(), capComm) diff --git a/pkg/rocket/getters.go b/pkg/rocket/getters.go index 01833e0..4b83950 100644 --- a/pkg/rocket/getters.go +++ b/pkg/rocket/getters.go @@ -23,6 +23,17 @@ type ( } ) +// All returns a copy of all the exported variables. +func (export exportMap) All() map[string]string { + m := make(map[string]string) + + for k, v := range export { + m[k] = v + } + + return m +} + type osEnvGetter struct{} // Gets an environment variable's value. diff --git a/pkg/rocket/include.go b/pkg/rocket/include.go index 4866b58..71e0b46 100644 --- a/pkg/rocket/include.go +++ b/pkg/rocket/include.go @@ -183,19 +183,59 @@ func loadPreMissionMaps(ctx context.Context, spaceDust map[string]interface{}, c return cfgMaps, nil } -func mergeMissions(mission, addition *Mission) { //nolint complexity - if addition.Name != "" { - mission.Name = addition.Name +func missionMergeParams(mission, addition *Mission) { + m := make(map[string]bool) + for _, p := range mission.Params { + m[p.Name] = true } - if addition.Version != "" { - mission.Version = addition.Version + + params := make([]Param, 0, len(addition.Params)) + for _, p := range addition.Params { + if _, ok := m[p.Name]; !ok || p.Name == "" { + params = append(params, p) + } + } + + mission.Params = append(mission.Params, params...) +} + +func missionMergeStages(mission, addition *Mission) { + m := make(map[string]bool) + for _, st := range mission.Stages { + m[st.Name] = true + } + + stages := make([]Stage, 0, len(addition.Stages)) + for _, st := range addition.Stages { + if _, ok := m[st.Name]; !ok || st.Name == "" { + stages = append(stages, st) + } + } + + mission.Stages = append(mission.Stages, stages...) +} + +func missionMergeSequences(mission, addition *Mission) { + if mission.Sequences == nil { + mission.Sequences = make(map[string][]string) + } + + for k, seq := range addition.Sequences { + if _, ok := mission.Sequences[k]; !ok { + mission.Sequences[k] = seq + } } +} + +func missionMergeEnv(mission, addition *Mission) { if len(addition.BasicEnv) > 0 { if mission.BasicEnv == nil { mission.BasicEnv = make(EnvMap) } for k, v := range addition.BasicEnv { - mission.BasicEnv[k] = v + if _, ok := mission.BasicEnv[k]; !ok { + mission.BasicEnv[k] = v + } } } if len(addition.Env) > 0 { @@ -203,29 +243,33 @@ func mergeMissions(mission, addition *Mission) { //nolint complexity mission.Env = make(EnvMap) } for k, v := range addition.Env { - mission.Env[k] = v + if _, ok := mission.Env[k]; !ok { + mission.Env[k] = v + } } } +} + +func mergeMissions(mission, addition *Mission) { + if mission.Name != "" { + mission.Name = addition.Name + } + if mission.Version != "" { + mission.Version = addition.Version + } + + missionMergeEnv(mission, addition) + if len(addition.Params) > 0 { - if mission.Params == nil { - mission.Params = make([]Param, 0, len(addition.Params)) - } - mission.Params = append(mission.Params, addition.Params...) + missionMergeParams(mission, addition) } + if len(addition.Stages) > 0 { - if mission.Stages == nil { - mission.Stages = make([]Stage, 0, len(addition.Stages)) - } - mission.Stages = append(mission.Stages, addition.Stages...) + missionMergeStages(mission, addition) } if len(addition.Sequences) > 0 { - if mission.Sequences == nil { - mission.Sequences = make(map[string][]string) - } - for k, v := range addition.Sequences { - mission.Sequences[k] = v - } + missionMergeSequences(mission, addition) } } diff --git a/pkg/rocket/io.go b/pkg/rocket/io.go index 38b686e..d44adf0 100644 --- a/pkg/rocket/io.go +++ b/pkg/rocket/io.go @@ -1,140 +1,21 @@ package rocket import ( - "fmt" - "os" + "github.com/nehemming/cirocket/pkg/providers" ) const ( // InputIO is the input file key. - InputIO = NamedIO("input") + InputIO = providers.ResourceID("input") // OutputIO is the output file key. - OutputIO = NamedIO("output") + OutputIO = providers.ResourceID("output") // ErrorIO is the error file key. - ErrorIO = NamedIO("error") + ErrorIO = providers.ResourceID("error") + + // Stdin is the Std in resource. + Stdin = providers.ResourceID("stdin") + // Stdout is the Std out resource. + Stdout = providers.ResourceID("stdout") + // Stderr is the Std error resource. + Stderr = providers.ResourceID("stderr") ) - -const ( - // IOModeInput file can be used for input. - IOModeInput = IOMode(1 << iota) - - // IOModeOutput file can be used for output. - IOModeOutput - - // IOModeError file can be used for errors. - IOModeError - - // IOModeTruncate file should be truncated. - IOModeTruncate - - // IOModeAppend file should be appended to. - IOModeAppend - - // IOModeNone is the default empty mde. - IOModeNone = IOMode(0) -) - -type ( - // IOMode indicates the modes of operation the file detail supports. - IOMode uint32 - - // NamedIO is a named io file. - NamedIO string - - // FileDetail represents file details. - FileDetail struct { - filePath string - ioMode IOMode - fileMode os.FileMode - } - - // ioSettings is a collection of the file and IO settings used by a task. - ioSettings struct { - files map[NamedIO]*FileDetail - } -) - -func newIOSettings() *ioSettings { - return &ioSettings{ - files: make(map[NamedIO]*FileDetail), - } -} - -// Creates a new copy from the parent. -func (ios *ioSettings) newCopy() *ioSettings { - copy := &ioSettings{ - files: make(map[NamedIO]*FileDetail), - } - - for k, v := range ios.files { - copy.files[k] = v - } - - return copy -} - -func (ios *ioSettings) addFilePath(name NamedIO, filePath string, mode IOMode) *FileDetail { - fd := &FileDetail{ - filePath: filePath, - ioMode: mode, - fileMode: 0o666, - } - - ios.files[name] = fd - - return fd -} - -func (ios *ioSettings) duplicate(from, to NamedIO) error { - if f, ok := ios.files[from]; ok { - ios.files[to] = f - return nil - } - return fmt.Errorf("file type %s could not be found", from) -} - -// getFileDetails returns the named file details or nil. -func (ios *ioSettings) getFileDetails(name NamedIO) *FileDetail { - return ios.files[name] -} - -func (fd *FileDetail) FilePath() string { - return fd.filePath -} - -// ReadFile reads the file into a byte slice or returns an error. -func (fd *FileDetail) ReadFile() ([]byte, error) { - if (fd.ioMode & IOModeInput) == IOModeNone { - return nil, fmt.Errorf("file type %s is nt an input file type", fd.filePath) - } - return os.ReadFile(fd.filePath) -} - -// InMode returns true if the file in in the mode in question. -func (fd *FileDetail) InMode(mode IOMode) bool { - return (fd.ioMode & mode) == mode -} - -// OpenOutput opens an output file. -func (fd *FileDetail) OpenOutput() (*os.File, error) { - if (fd.ioMode & (IOModeOutput | IOModeError)) == IOModeNone { - return nil, fmt.Errorf("file type %s is nt an output file type", fd.filePath) - } - - if (fd.ioMode & IOModeTruncate) == IOModeTruncate { - return os.OpenFile(fd.filePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, fd.fileMode) - } else if (fd.ioMode & IOModeAppend) == IOModeAppend { - return os.OpenFile(fd.filePath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, fd.fileMode) - } else { - return nil, fmt.Errorf("file type %s does not specify create mode", fd.filePath) - } -} - -// OpenInput opens the file for input. -func (fd *FileDetail) OpenInput() (*os.File, error) { - if (fd.ioMode & IOModeInput) == IOModeNone { - return nil, fmt.Errorf("file type %s is nt an input file type", fd.filePath) - } - - return os.OpenFile(fd.filePath, os.O_RDONLY, fd.fileMode) -} diff --git a/pkg/rocket/io_test.go b/pkg/rocket/io_test.go deleted file mode 100644 index 63c483e..0000000 --- a/pkg/rocket/io_test.go +++ /dev/null @@ -1,118 +0,0 @@ -package rocket - -import ( - "fmt" - "testing" -) - -func TestIOModes(t *testing.T) { - r := modeString(IOModeNone) - if r != "0:000000" { - t.Error("IOModeNone", r) - } - - r = modeString(IOModeInput) - if r != "1:000001" { - t.Error("IOModeInput", r) - } - - r = modeString(IOModeOutput) - if r != "2:000010" { - t.Error("IOModeOutput", r) - } - - r = modeString(IOModeError) - if r != "4:000100" { - t.Error("IOModeError", r) - } - - r = modeString(IOModeTruncate) - if r != "8:001000" { - t.Error("IOModeCreate", r) - } - - r = modeString(IOModeAppend) - if r != "16:010000" { - t.Error("IOModeCreate", r) - } -} - -func modeString(mode IOMode) string { - return fmt.Sprintf("%d:%06b", mode, mode) -} - -func TestNewCopy(t *testing.T) { - ios := newIOSettings() - - ios.addFilePath(OutputIO, "test123", IOModeOutput) - - copy := ios.newCopy() - - fd := copy.getFileDetails(OutputIO) - - if fd == nil || fd.filePath != "test123" || fd.ioMode != IOModeOutput { - t.Error("Copy missing correct data", fd) - } -} - -func TestDuplicateErrorsOnMissing(t *testing.T) { - ios := newIOSettings() - - if err := ios.duplicate(OutputIO, ErrorIO); err == nil { - t.Error("duplicate no error on missing source") - } -} - -func TestReadFileFailsWrongType(t *testing.T) { - ios := newIOSettings() - fd := ios.addFilePath(OutputIO, "context.go", IOModeOutput) - - if _, err := fd.ReadFile(); err == nil { - t.Error("No error on output read") - } -} - -func TestReadFileSucceedsForIInput(t *testing.T) { - ios := newIOSettings() - fd := ios.addFilePath(InputIO, "context.go", IOModeInput) - - if b, err := fd.ReadFile(); err != nil { - t.Error("Error on output read", err) - } else if len(b) < 200 { - t.Error("Error context.go too small when read", len(b)) - } -} - -func TestOpenInputFailsWrongType(t *testing.T) { - ios := newIOSettings() - fd := ios.addFilePath(InputIO, "context.go", IOModeOutput) - - if _, err := fd.OpenInput(); err == nil { - t.Error("No error on output read") - } -} - -func TestOpenInputSucceedsForIInput(t *testing.T) { - ios := newIOSettings() - fd := ios.addFilePath(InputIO, "context.go", IOModeInput) - - if f, err := fd.OpenInput(); err != nil { - t.Error("Error on output read", err) - } else { - f.Close() - } -} - -func TestOpenOutput(t *testing.T) { - ios := newIOSettings() - fd := ios.addFilePath(InputIO, "dummy.txt", IOModeInput) - - if _, err := fd.OpenOutput(); err == nil { - t.Error("no error opening input as output") - } - - fd = ios.addFilePath(InputIO, "dummy.txt", IOModeOutput) - if _, err := fd.OpenOutput(); err == nil { - t.Error("no error opening output no mode") - } -} diff --git a/pkg/rocket/missioncontrol.go b/pkg/rocket/missioncontrol.go index acb4484..b4d21ad 100644 --- a/pkg/rocket/missioncontrol.go +++ b/pkg/rocket/missioncontrol.go @@ -41,6 +41,16 @@ type ( // the coonfig file is the source name iof the config provided // if its empty the current working 'dir/default' will be used. LaunchMission(ctx context.Context, configFile string, spaceDust map[string]interface{}, flightSequences ...string) error + + // LaunchMissionWithParams loads and executes the mission with user supplied parameters + // flightSequences may be specified, each sequence is run in the order specified + // the coonfig file is the source name iof the config provided + // if its empty the current working 'dir/default' will be used. + // The supplied params are default values and do not override values defined in the mission + LaunchMissionWithParams(ctx context.Context, configFile string, + spaceDust map[string]interface{}, + params []Param, + flightSequences ...string) error } // operations is a collection of operations. @@ -84,6 +94,12 @@ func (mc *missionControl) RegisterTaskTypes(types ...TaskType) { } func (mc *missionControl) LaunchMission(ctx context.Context, configFile string, spaceDust map[string]interface{}, flightSequences ...string) error { + return mc.LaunchMissionWithParams(ctx, configFile, spaceDust, nil, flightSequences...) +} + +func (mc *missionControl) LaunchMissionWithParams(ctx context.Context, configFile string, + spaceDust map[string]interface{}, params []Param, + flightSequences ...string) error { configFile, err := getConfigFileName(configFile) if err != nil { return err @@ -96,7 +112,7 @@ func (mc *missionControl) LaunchMission(ctx context.Context, configFile string, } // Create a cap comm object from the environment - capComm := newCapCommFromEnvironment(configFile) + capComm := newCapCommFromEnvironment(configFile, loggee.Default()) // Misssion has been successfully parsed, load the global settings capComm, err = processGlobals(ctx, capComm, mission) diff --git a/pkg/rocket/missioncontrol_test.go b/pkg/rocket/missioncontrol_test.go index 0ea1137..d49c7a7 100644 --- a/pkg/rocket/missioncontrol_test.go +++ b/pkg/rocket/missioncontrol_test.go @@ -13,6 +13,7 @@ import ( ) func TestNewMissionControl(t *testing.T) { + loggee.SetLogger(stdlog.New()) mc := NewMissionControl() if mc == nil { @@ -61,6 +62,7 @@ func (tt *testTaskType) Prepare(ctx context.Context, capComm *CapComm, task Task } func TestRegisterTaskTypes(t *testing.T) { + loggee.SetLogger(stdlog.New()) mc := NewMissionControl() mc.RegisterTaskTypes() @@ -85,6 +87,7 @@ func TestRegisterTaskTypes(t *testing.T) { } func TestLaunchMissionZero(t *testing.T) { + loggee.SetLogger(stdlog.New()) mc := NewMissionControl() if err := mc.LaunchMission(context.Background(), "", nil); err != nil { @@ -93,6 +96,7 @@ func TestLaunchMissionZero(t *testing.T) { } func TestLaunchMissionOne(t *testing.T) { + loggee.SetLogger(stdlog.New()) mc := NewMissionControl() mission := make(map[string]interface{}) @@ -104,6 +108,7 @@ func TestLaunchMissionOne(t *testing.T) { } func TestLaunchMissionTwo(t *testing.T) { + loggee.SetLogger(stdlog.New()) mc := NewMissionControl() mission := make(map[string]interface{}) diff --git a/pkg/rocket/variablewriter.go b/pkg/rocket/variablewriter.go new file mode 100644 index 0000000..e54a9d5 --- /dev/null +++ b/pkg/rocket/variablewriter.go @@ -0,0 +1,39 @@ +package rocket + +import ( + "bytes" + "context" + "io" + + "github.com/pkg/errors" +) + +type variableWriter struct { + data bytes.Buffer + name string + capComm *CapComm +} + +func newVariableWriter(capComm *CapComm, name string) *variableWriter { + return &variableWriter{ + capComm: capComm, + name: name, + } +} + +func (vw *variableWriter) OpenRead(ctx context.Context) (io.ReadCloser, error) { + return nil, errors.New("variables cannot be read") +} + +func (vw *variableWriter) OpenWrite(ctx context.Context) (io.WriteCloser, error) { + return vw, nil +} + +func (vw *variableWriter) Write(p []byte) (n int, err error) { + return vw.data.Write(p) +} + +func (vw *variableWriter) Close() error { + vw.capComm.ExportVariable(vw.name, vw.data.String()) + return nil +}