diff --git a/cli/cloud/LICENSE b/cli/cloud/LICENSE new file mode 100644 index 0000000000..8ce81b0f35 --- /dev/null +++ b/cli/cloud/LICENSE @@ -0,0 +1,391 @@ +Tracetest Community License Agreement + +Please read this Tracetest Community License Agreement (the “Agreement”) +carefully before using Tracetest (as defined below), which is offered by +Tracetest or its affiliated Legal Entities (“Tracetest”). + +By accessing, installing, downloading or in any manner using Tracetest, +You agree that You have read and agree to be bound by the terms of this +Agreement. If You are accessing Tracetest on behalf of a Legal Entity, +You represent and warrant that You have the authority to agree to these +terms on its behalf and the right to bind that Legal Entity to this +Agreement. Use of Tracetest is expressly conditioned upon Your assent to +all the terms of this Agreement, as well as the other Tracetest agreements, +including the Tracetest Privacy Policy and Tracetest Terms and Conditions, +accessible at: https://app.tracetest.io/privacy-policy.html and +https://app.tracetest.io/terms-of-service.html. + +1. Definitions. In addition to other terms defined elsewhere in this Agreement +and in the other Tracetest agreements, the terms below have the following +meanings. + +(a) “Tracetest” shall mean the Test Orchestration and Execution software +provided by Tracetest, including both Tracetest Core and Tracetest Pro, as +defined below. + +(b) “Tracetest Core” shall mean the version and features of Tracetest designated +as free of charge at https://tracetest.io/pricing and available at +https://github.com/kubeshop/tracetest pursuant to the terms of the MIT license. + +(c) “Tracetest Pro” shall mean the version of Tracetest which includes the +additional paid features of Tracetest designated at +https://tracetest.io/pricing and made available by Tracetest, also at +https://github.com/kubeshop/tracetest, the use of which is subject to additional +terms set out below. + +(d) “Contribution” shall mean any work of authorship, including the original +version of the Work and any modifications or additions to that Work or +Derivative Works thereof, that is intentionally submitted to Tracetest for +inclusion in the Work by the copyright owner or by an individual or Legal Entity +authorized to submit on behalf of the copyright owner. For the purposes of this +definition, “submitted” means any form of electronic, verbal, or written +communication sent to Tracetest or its representatives, including but not +limited to communication on electronic mailing lists, source code control +systems, and issue tracking systems that are managed by, or on behalf of, +Tracetest for the purpose of discussing and improving the Work, but excluding +communication that is conspicuously marked or otherwise designated in writing +by the copyright owner as “Not a Contribution.” + +(e) “Contributor” shall mean any copyright owner or individual or Legal Entity +authorized by the copyright owner, other than Tracetest, from whom Tracetest +receives a Contribution that Tracetest subsequently incorporates within the Work. + +(f) “Derivative Works” shall mean any work, whether in Source or Object form, +that is based on (or derived from) the Work, such as a translation, abridgement, +condensation, or any other recasting, transformation, or adaptation for which +the editorial revisions, annotations, elaborations, or other modifications +represent, as a whole, an original work of authorship. For the purposes of this +License, Derivative Works shall not include works that remain separable from, or +merely link (or bind by name) to the interfaces of, the Work and Derivative +Works thereof. You may create certain Derivative Works of Tracetest Pro (“Pro +Derivative Works”, as defined below) provided that such Pro Derivative Works +are solely created, distributed, and accessed for Your internal use, and are +not created, distributed, or accessed in such a way that the Pro Derivative +Works would modify, circumvent, or otherwise bypass controls implemented, if +any, to ensure that Tracetest Pro users comply with the terms of the Paid Pro +License. Notwithstanding anything contained herein to the contrary, You may not +modify or alter the Source of Tracetest Pro absent Tracetest’s prior express +written permission. If You have any questions about creating Pro Derivative +Works or otherwise modifying or redistributing Tracetest Pro, please contact +Tracetest at support@Tracetest.io. + +(g) “Legal Entity” shall mean the union of the acting entity and all other +entities that control, are controlled by, or are under common control with that +entity. For the purposes of this definition, “control” means (i) the power, +direct or indirect, to cause the direction or management of such entity, whether +by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of +the outstanding shares, or (iii) beneficial ownership of such entity. + +(h) “License” shall mean the terms and conditions for use, reproduction, and +distribution of a Work as defined by this Agreement. + +(i) “Licensor” shall mean Tracetest or a Contributor, as applicable. + +(j) “Object” form shall mean any form resulting from mechanical transformation +or translation of a Source form, including but not limited to compiled object +code, generated documentation, and conversions to other media types. + +(k) “Source” form shall mean the preferred form for making modifications, +including but not limited to software source code, documentation source, and +configuration files. + +(l) “Third Party Works” shall mean Works, including Contributions, and other +technology owned by a person or Legal Entity other than Tracetest, as indicated +by a copyright notice that is included in or attached to such Works or technology. + +(m) “Work” shall mean the work of authorship, whether in Source or Object form, +made available under a License, as indicated by a copyright notice that is +included in or attached to the work. + +(n) “You” (or “Your”) shall mean an individual or Legal Entity exercising +permissions granted by this License. + +2. Licenses. + +(a) License to Tracetest Core. The License for the applicable version of +Tracetest Core can be found on the Tracetest Licensing FAQs page and in the +applicable license file within the Tracetest GitHub repository(ies). Tracetest +Core is a no-cost, entry-level license and as such, contains the following +disclaimers: NOTWITHSTANDING ANYTHING TO THE CONTRARY HEREIN, Tracetest CORE +IS PROVIDED “AS IS” AND “AS AVAILABLE”, AND ALL EXPRESS OR IMPLIED WARRANTIES +ARE EXCLUDED AND DISCLAIMED, INCLUDING WITHOUT LIMITATION THE IMPLIED WARRANTIES +OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND ANY +WARRANTIES ARISING BY STATUTE OR OTHERWISE IN LAW OR FROM COURSE OF DEALING, +COURSE OF PERFORMANCE, OR USE IN TRADE. For clarity, the terms of this Agreement, +other than the relevant definitions in Section 1 and this Section 2(a) do not +apply to Tracetest Core. + +(b) License to Tracetest Pro. + +(i) Grant of Copyright License: Subject to the terms of this Agreement, Licensor +hereby grants to You a worldwide, non-exclusive, non-transferable limited +license to reproduce, prepare Pro Derivative Works (as defined below) of, +publicly display, publicly perform, sublicense, and distribute Tracetest Pro for +Your business purposes, for so long as You are not in violation of this +Section 2(b) and are current on all payments required by Section 4 below. + +(ii) Grant of Patent License: Subject to the terms of this Agreement, Licensor +hereby grants to You a worldwide, non-exclusive, non-transferable limited patent +license to make, have made, use, and import Tracetest Pro, where such license +applies only to those patent claims licensable by Licensor that are necessarily +infringed by their Contribution(s) alone or by combination of their +Contribution(s) with the Work to which such Contribution(s) was submitted. If You +institute patent litigation against any entity (including a cross-claim or +counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated +within the Work constitutes direct or contributory patent infringement, then any +patent licenses granted to You under this License for that Work shall terminate +as of the date such litigation is filed. + +(iii) License to Third Party Works: From time to time Tracetest may use, or +provide You access to, Third Party Works in connection with Tracetest Pro. You +acknowledge and agree that in addition to this Agreement, Your use of Third Party +Works is subject to all other terms and conditions set forth in the License +provided with or contained in such Third Party Works. Some Third Party Works may +be licensed to You solely for use with Tracetest Pro under the terms of a third +party License, or as otherwise notified by Tracetest, and not under the terms of +this Agreement. You agree that the owners and third party licensors of Third +Party Works are intended third party beneficiaries to this Agreement, and You +agree to abide by all third party terms and conditions and licenses. + +3. Support. From time to time, in its sole discretion, Tracetest may offer +professional services or support for Tracetest, which may now or in the future be +subject to additional fees, as outlined at https://tracetest.io/pricing. + +4. Fees for Tracetest Pro or Tracetest Support. + +(a) Fees. The License to Tracetest Pro is conditioned upon Your entering into a +subscription agreement with Tracetest for its use (a “Paid Pro License”) and +timely paying Tracetest for such Paid Pro License; provided that features of +Tracetest Pro that are features of Tracetest Core and are not designated as “Pro +features” at https://tracetest.io/pricing may be used for free under the terms of +the Agreement without a Paid Pro License. Tracetest Pro may at its discretion +include within Tracetest Pro certain Source code solely intended to determine +Your compliance with the Paid Pro License which may be accessed without a Paid +Pro License, provided that under no circumstances may You modify Tracetest Pro +to circumvent the Paid Pro License requirement. Any professional services or +support for Tracetest may also be subject to Your payment of fees, which will be +specified by Tracetest when you sign up to receive such professional services or +support. Tracetest reserves the right to change the fees at any time with prior +written notice; for recurring fees, any such adjustments will take effect as of +the next pay period. + +(b) Overdue Payments and Taxes. Overdue payments are subject to a service charge +equal to the lesser of 1.5% per month or the maximum legal interest rate allowed +by law, and You shall pay all Tracetest reasonable costs of collection, including +court costs and attorneys’ fees. Fees are stated and payable in U.S. dollars and +are exclusive of all sales, use, value added and similar taxes, duties, +withholdings and other governmental assessments (but excluding taxes based on +Tracetest income) that may be levied on the transactions contemplated by this +Agreement in any jurisdiction, all of which are Your responsibility unless you +have provided Tracetest with a valid tax-exempt certificate. If You owe Tracetest +overdue payments, Tracetest reserves the right to revoke any license(s) granted +by this Agreement and revoke to Your access to Tracetest Core and to Tracetest Pro. + +(c) Record-keeping and Audit. If fees for Tracetest Pro are based on the number +of environments running on Tracetest Pro or another use-based unit of measurement, +including number of users, You must maintain complete and accurate records with +respect Your use of Tracetest Pro and will provide such records to Tracetest for +inspection or audit upon Tracetest’s reasonable request. If an inspection or +audit uncovers additional usage by You for which fees are owed under this +Agreement, then You shall pay for such additional usage at Tracetest’s +then-current rates. + +5. Trial License. If You have signed up for a trial or evaluation of Tracetest +Pro, Your License to Tracetest Pro is granted without charge for the trial or +evaluation period specified when You signed up, or if no term was specified, for +forty-five (45) calendar days, provided that Your License is granted solely for +purposes of Your internal evaluation of Tracetest Pro during the trial or +evaluation period (a “Trial License”). You may not use Tracetest Pro or any +Tracetest Pro features under a Trial License more than once in any twelve (12) +month period. Tracetest may revoke a Trial License at any time and for any reason. +Sections 3, 4, 9 and 11 of this Agreement do not apply to Trial Licenses. + +6. Redistribution. You may reproduce and distribute copies of the Work or +Derivative Works thereof in any medium, with or without modifications, and in +Source or Object form, provided that You meet the following conditions: + +(a) You must give any other recipients of the Work or Derivative Works a copy of +this License; and + +(b) You must cause any modified files to carry prominent notices stating that +You changed the files; and + +(c) You must retain, in the Source form of any Derivative Works that You +distribute, including for internal purposes at Your Legal Entities, all +copyright, patent, trademark, and attribution notices from the Source form of +the Work, excluding those notices that do not pertain to any part of the +Derivative Works; and + +(d) If the Work includes a “NOTICE” or equivalent text file as part of its +distribution, then any Derivative Works that You distribute must include a +readable copy of the attribution notices contained within such NOTICE file, +excluding those notices that do not pertain to any part of the Derivative Works, +in at least one of the following places: within a NOTICE text file distributed +as part of the Derivative Works; within the Source form or documentation, if +provided along with the Derivative Works; or, within a display generated by the +Derivative Works, if and wherever such third-party notices normally appear. The +contents of the NOTICE or equivalent files are for informational purposes only +and do not modify the License. You may add Your own attribution notices within +Derivative Works that You distribute for Your internal use, alongside or as an +addendum to the NOTICE text from the Work, provided that such additional +attribution notices cannot be construed as modifying the License. + +You may not create Derivative Works, including Pro Derivative Works (as defined +below), which add Your own copyright statements or provide additional or +different license terms and conditions for use, reproduction, or distribution of +Your modifications, or for any such Derivative Works as a whole. All Derivative +Works, including Your use, reproduction, and distribution of the Work, must +comply in all respects with the conditions stated in this License. + +(e) Pro Derivative Works: Derivative Works of Tracetest Pro (“Pro Derivative +Works”) may only be made, reproduced and distributed, without modifications, in +Source or Object form, provided that such Pro Derivative Works are solely for +Your internal use. Each Pro Derivative Work shall be governed by this Agreement, +shall include a License to Tracetest Pro, and thus will be subject to the payment +of fees to Tracetest by any user of the Pro Derivative Work. + +7. Submission of Contributions. Unless You explicitly state otherwise, any +Contribution submitted for inclusion in Tracetest Pro by You to Tracetest shall be +under the terms and conditions of this Agreement, without any additional terms +or conditions, payments of royalties or otherwise to Your benefit. Tracetest may +at any time, at its sole discretion, elect for the Contribution to be subject to +the Paid Pro License. If You wish to reserve any rights regarding Your +Contribution, You must contact Tracetest at support@Tracetest.io prior to +submitting the Contribution. + +8. Trademarks. This License does not grant permission to use the trade names, +trademarks, service marks, or product names of Licensor, except as required for +reasonable and customary use in describing the origin of the Work and reproducing +the content of the NOTICE or equivalent file. + +9. Limited Warranty. + +(a) Warranties. Subject to the terms of the Paid Pro License, or any other +agreement between You and Tracetest which governs the terms of Your access to +Tracetest Pro, Tracetest warrants to You that: (i) Tracetest Pro will materially +perform in accordance with the applicable documentation for thirty (30) days +after initial delivery to You; and (ii) any professional services performed by +Tracetest under this Agreement will be performed in a workmanlike manner, in +accordance with general industry standards. + +(b) Exclusions. Tracetest’s warranties in this Section 9 do not extend to problems +that result from: (i) Your failure to implement updates issued by Tracetest during +the warranty period; (ii) any alterations or additions (including Pro Derivative +Works and Contributions) to Tracetest not performed by or at the direction of +Tracetest; (iii) failures that are not reproducible by Tracetest; (iv) operation +of Tracetest Pro in violation of this Agreement or not in accordance with its +documentation; (v) failures caused by software, hardware, or products not +licensed or provided by Tracetest hereunder; or (vi) Third Party Works. + +(c) Remedies. In the event of a breach of a warranty under this Section 9, +Tracetest will, at its discretion and cost, either repair, replace or re-perform +the applicable Works or services or refund a portion of fees previously paid to +Tracetest that are associated with the defective Works or services. This is Your +exclusive remedy, and Tracetest’s sole liability, arising in connection with the +limited warranties herein and shall, in all cases, be limited to the fees paid +to Tracetest in the three (3) months preceding the delivery of the defective Works +or services. + +10. Disclaimer of Warranty. Except as set out in Section 9, unless required by +applicable law, Licensor provides the Work (and each Contributor provides its +Contributions) on an “AS IS” BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +either express or implied, arising out of course of dealing, course of +performance, or usage in trade, including, without limitation, any warranties +or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, CORRECTNESS, +RELIABILITY, or FITNESS FOR A PARTICULAR PURPOSE, all of which are hereby +disclaimed. You are solely responsible for determining the appropriateness of +using or redistributing Works and assume any risks associated with Your exercise +of permissions under the applicable License for such Works. + +11. Limited Indemnity. + +(a) Indemnity. Tracetest will defend, indemnify and hold You harmless against +any third party claims, liabilities or expenses incurred (including reasonable +attorneys’ fees), as well as amounts finally awarded in a settlement or a +non-appealable judgement by a court (“Losses”), to the extent arising from any +claim or allegation by a third party that Tracetest Pro infringes or +misappropriates a valid United States patent, copyright, or trade secret right +of a third party; provided that You give Tracetest: (i) prompt written notice of +any such claim or allegation; (ii) sole control of the defense and settlement +thereof; and (iii) reasonable cooperation and assistance in such defense or +settlement. If any Work within Tracetest Pro becomes or in Tracetest’s opinion is +likely to become, the subject of an injunction, Tracetest may, at its option, +(A) procure for You the right to continue using such Work, (B) replace or modify +such Work so that it becomes non-infringing without substantially compromising +its functionality, or, if (A) and (B) are not commercially practicable, then (C) +terminate Your license to the allegedly infringing Work and refund to You a +prorated portion of the prepaid and unearned fees for such infringing Work. The +foregoing comprises the entire liability of Tracetest with respect to infringement +of patents, copyrights, trade secrets, or other intellectual property rights. + +(b) Exclusions. The foregoing obligations on Tracetest shall not apply to: (i) +Works modified by any party other than Tracetest (including Pro Derivative Works +and Contributions) where the alleged infringement relates to such modification, +(ii) Works combined or bundled with any products, processes, or materials not +provided by Tracetest where the alleged infringement relates to such combination, +(iii) use of a version of Tracetest Pro other than the version that was current at +the time of such use, as long as a non-infringing version had been released at +the time of the alleged infringement, (iv) any Works created to Your +specifications, (v) infringement or misappropriation of any proprietary or +intellectual property right in which You have an interest, or (vi) Third Party +Works. You will defend, indemnify, and hold Tracetest harmless against any Losses +arising from any such claim or allegation as described in the scenarios in this +Section 11(b), subject to conditions reciprocal to those in Section 11(a). + +12. Limitation of Liability. In no event and under no legal or equitable theory, +whether in tort (including negligence), contract, or otherwise, unless required +by applicable law (such as deliberate and grossly negligent acts), and +notwithstanding anything in this Agreement to the contrary, shall Licensor or +any Contributor be liable to You for (i) any amounts in excess, in the aggregate, +of the fees paid by You to Tracetest under this Agreement in the twelve (12) +months preceding the date the first cause of liability arose, or (ii) any +indirect, special, incidental, punitive, exemplary, reliance, or consequential +damages of any character arising as a result of this Agreement or out of the use +or inability to use the Work (including but not limited to damages for loss of +goodwill, profits, data or data use, work stoppage, computer failure or +malfunction, cost of procurement of substitute goods, technology or services, +or any and all other commercial damages or losses), even if such Licensor or +Contributor has been advised of the possibility of such damages. THESE +LIMITATIONS SHALL APPLY NOTWITHSTANDING THE FAILURE OF THE ESSENTIAL PURPOSE OF +ANY LIMITED REMEDY. + +13. General. + +(a) Relationship of Parties. You and Tracetest are independent contractors, and +nothing herein shall be deemed to constitute either party as the agent or +representative of the other or both parties as joint venturers or partners for +any purpose. + +(b) Export Control. You shall comply with the U.S. Foreign Corrupt Practices Act +and all applicable export laws, restrictions and regulations of the U.S. +Department of Commerce, U.S. Department of Treasury, and any other applicable +U.S. and foreign authority(ies). + +(c) Assignment. This Agreement and the rights and obligations herein may not be +assigned or transferred, in whole or in part, by You without the prior written +consent of Tracetest. Any assignment in violation of this provision is void. This +Agreement shall be binding upon, and inure to the benefit of, the successors and +permitted assigns of the parties. + +(d) Governing Law. This Agreement shall be governed by and construed under the +laws of the State of Delaware and the United States without regard to conflicts +of laws provisions thereof, and without regard to the Uniform Computer +Information Transactions Act. + +(e) Attorneys’ Fees. In any action or proceeding to enforce rights under this +Agreement, the prevailing party shall be entitled to recover its costs, expenses, +and attorneys’ fees. + +(f) Severability. If any provision of this Agreement is held to be invalid, +illegal, or unenforceable in any respect, that provision shall be limited or +eliminated to the minimum extent necessary so that this Agreement otherwise +remains in full force and effect and enforceable. + +(g) Entire Agreement; Waivers; Modification. This Agreement constitutes the +entire agreement between the parties relating to the subject matter hereof and +supersedes all proposals, understandings, or discussions, whether written or +oral, relating to the subject matter of this Agreement and all past dealing or +industry custom. The failure of either party to enforce its rights under this +Agreement at any time for any period shall not be construed as a waiver of such +rights. No changes, modifications or waivers to this Agreement will be effective +unless in writing and signed by both parties. \ No newline at end of file diff --git a/cli/cloud/cmd/run_cmd.go b/cli/cloud/cmd/run_cmd.go new file mode 100644 index 0000000000..e2492d70c5 --- /dev/null +++ b/cli/cloud/cmd/run_cmd.go @@ -0,0 +1,63 @@ +package cmd + +import ( + "context" + "fmt" + + "github.com/kubeshop/tracetest/cli/cloud/runner" + "github.com/kubeshop/tracetest/cli/cmdutil" + "github.com/kubeshop/tracetest/cli/config" + "github.com/kubeshop/tracetest/cli/formatters" + "github.com/kubeshop/tracetest/cli/pkg/resourcemanager" + "github.com/kubeshop/tracetest/cli/preprocessor" + + cliRunner "github.com/kubeshop/tracetest/cli/runner" +) + +func RunMultipleFiles(ctx context.Context, runParams *cmdutil.RunParameters, cliConfig *config.Config, runnerRegistry cliRunner.Registry, format string) (int, error) { + if cliConfig.Jwt == "" { + return cliRunner.ExitCodeGeneralError, fmt.Errorf("you should be authenticated to run multiple files, please run 'tracetest configure'") + } + + variableSetPreprocessor := preprocessor.VariableSet(cmdutil.GetLogger()) + + formatter := formatters.MultipleRun[cliRunner.RunResult](func() string { return cliConfig.UI() }, true) + + orchestrator := runner.MultiFileOrchestrator( + cmdutil.GetLogger(), + config.GetAPIClient(*cliConfig), + GetVariableSetClient(variableSetPreprocessor), + runnerRegistry, + formatter, + ) + + return orchestrator.Run(ctx, runner.RunOptions{ + DefinitionFiles: runParams.DefinitionFiles, + VarsID: runParams.VarsID, + SkipResultWait: runParams.SkipResultWait, + JUnitOuptutFile: runParams.JUnitOuptutFile, + RequiredGates: runParams.RequiredGates, + RunGroupID: runParams.RunGroupID, + }, format) +} + +func GetVariableSetClient(preprocessor preprocessor.Preprocessor) resourcemanager.Client { + httpClient := &resourcemanager.HTTPClient{} + + variableSetClient := resourcemanager.NewClient( + httpClient, cmdutil.GetLogger(), + "variableset", "variablesets", + resourcemanager.WithTableConfig(resourcemanager.TableConfig{ + Cells: []resourcemanager.TableCellConfig{ + {Header: "ID", Path: "spec.id"}, + {Header: "NAME", Path: "spec.name"}, + {Header: "DESCRIPTION", Path: "spec.description"}, + }, + }), + resourcemanager.WithResourceType("VariableSet"), + resourcemanager.WithApplyPreProcessor(preprocessor.Preprocess), + resourcemanager.WithDeprecatedAlias("Environment"), + ) + + return variableSetClient +} diff --git a/cli/cloud/runner/multifile_orchestrator.go b/cli/cloud/runner/multifile_orchestrator.go new file mode 100644 index 0000000000..93fbbb9b82 --- /dev/null +++ b/cli/cloud/runner/multifile_orchestrator.go @@ -0,0 +1,374 @@ +package runner + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "os" + "sync" + "time" + + "github.com/davecgh/go-spew/spew" + "github.com/kubeshop/tracetest/cli/formatters" + "github.com/kubeshop/tracetest/cli/metadata" + "github.com/kubeshop/tracetest/cli/openapi" + "github.com/kubeshop/tracetest/cli/pkg/fileutil" + "github.com/kubeshop/tracetest/cli/pkg/resourcemanager" + "github.com/kubeshop/tracetest/cli/runner" + "github.com/kubeshop/tracetest/cli/varset" + "github.com/kubeshop/tracetest/server/pkg/id" + "go.uber.org/zap" +) + +// RunOptions defines options for running a resource +// IDs and DefinitionFiles are mutually exclusive and the only required options +type RunOptions struct { + // path to the file with resource definition + // the file will be applied before running + DefinitionFiles []string + + // varsID or path to the file with environment definition + VarsID string + + // runGroupID as string to define it for the entire run execution + RunGroupID string + + // By default the runner will wait for the result of the run + // if this option is true, the wait will be skipped + SkipResultWait bool + + // Optional path to the file where the result of the run will be saved + // in JUnit xml format + JUnitOuptutFile string + + // Overrides the default required gates for the resource + RequiredGates []string +} + +type MultipleRunFormatter interface { + Format(output formatters.MultipleRunOutput[runner.RunResult], format formatters.Output) string +} + +func MultiFileOrchestrator( + logger *zap.Logger, + openapiClient *openapi.APIClient, + variableSets resourcemanager.Client, + runnerRegistry runner.Registry, + multipleRunFormatter MultipleRunFormatter, +) orchestrator { + return orchestrator{ + logger: logger, + openapiClient: openapiClient, + variableSets: variableSets, + runnerRegistry: runnerRegistry, + multipleRunFormatter: multipleRunFormatter, + } +} + +type orchestrator struct { + logger *zap.Logger + + openapiClient *openapi.APIClient + variableSets resourcemanager.Client + runnerRegistry runner.Registry + multipleRunFormatter MultipleRunFormatter +} + +const ( + ExitCodeSuccess = 0 + ExitCodeGeneralError = 1 + ExitCodeTestNotPassed = 2 +) + +func (o orchestrator) Run(ctx context.Context, opts RunOptions, outputFormat string) (exitCode int, _ error) { + o.logger.Debug( + "Running tests from definition", + zap.Strings("definitionFiles", opts.DefinitionFiles), + zap.String("varSetID", opts.VarsID), + zap.Bool("skipResultsWait", opts.SkipResultWait), + zap.String("junitOutputFile", opts.JUnitOuptutFile), + zap.Strings("requiredGates", opts.RequiredGates), + ) + + variableSetFetcher := runner.GetVariableSetFetcher(o.logger, o.variableSets) + + varsID, err := variableSetFetcher.Fetch(ctx, opts.VarsID) + if err != nil { + return ExitCodeGeneralError, fmt.Errorf("cannot resolve variable set id: %w", err) + } + o.logger.Debug("env resolved", zap.String("ID", varsID)) + + hasDefinitionFilesDefined := opts.DefinitionFiles != nil && len(opts.DefinitionFiles) > 0 + resourceFetcher := runner.GetResourceFetcher(o.logger, o.runnerRegistry) + + if !hasDefinitionFilesDefined { + return ExitCodeGeneralError, fmt.Errorf("you must define at least two files to use the multifile orchestrator") + } + + vars := varset.VarSets{} + var resources []any + + runGroupID := opts.RunGroupID + if runGroupID == "" { + runGroupID = id.GenerateID().String() + } + runsResults := make([]runner.RunResult, 0) + definitionFiles, err := o.getDefinitionFiles(opts.DefinitionFiles) + + if err != nil { + return ExitCodeGeneralError, fmt.Errorf("cannot read definition files: %w", err) + } + + // 1. create runs + for _, definitionFile := range definitionFiles { + result, resource, err := o.createRun(ctx, resourceFetcher, &vars, opts.RequiredGates, definitionFile, varsID, runGroupID) + if err != nil { + return ExitCodeGeneralError, fmt.Errorf("cannot run test: %w", err) + } + + runsResults = append(runsResults, result) + resources = append(resources, resource) + } + + runnerGetter := func(resource any) (formatters.Runner[runner.RunResult], error) { + resourceType, err := resourcemanager.GetResourceType(resource) + if err != nil { + return nil, fmt.Errorf("cannot extract type from resource: %w", err) + } + + runner, err := o.runnerRegistry.Get(resourceType) + if err != nil { + return nil, fmt.Errorf("cannot find runner for resource type %s: %w", resourceType, err) + } + + return runner, nil + } + + // 3. if skip wait, print results and exit + if opts.SkipResultWait { + output := formatters.MultipleRunOutput[runner.RunResult]{ + Runs: runsResults, + Resources: resources, + RunGroup: openapi.RunGroup{Id: runGroupID}, + RunnerGetter: runnerGetter, + HasResults: false, + } + + summary := o.multipleRunFormatter.Format(output, formatters.Output(outputFormat)) + fmt.Println(summary) + return ExitCodeSuccess, nil + } + + // 4. wait for the run group + runGroup, err := o.waitForRunGroup(ctx, runGroupID) + if err != nil { + return ExitCodeGeneralError, fmt.Errorf("cannot wait for test result: %w", err) + } + + // 5. update runs and print results + for i, result := range runsResults { + resource := resources[i] + + resourceType, err := resourcemanager.GetResourceType(resource) + if err != nil { + return ExitCodeGeneralError, fmt.Errorf("cannot extract type from resource: %w", err) + } + + runner, err := o.runnerRegistry.Get(resourceType) + if err != nil { + return ExitCodeGeneralError, fmt.Errorf("cannot find runner for resource type %s: %w", resourceType, err) + } + + // TODO: I think we can just pull the test runs from the group instead of updating each of them + updated, err := runner.UpdateResult(ctx, result) + if err != nil { + return ExitCodeGeneralError, fmt.Errorf("cannot update test result: %w", err) + } + runsResults[i] = updated + + err = o.writeJUnitReport(ctx, runner, result, opts.JUnitOuptutFile) + if err != nil { + return ExitCodeGeneralError, fmt.Errorf("cannot write junit report: %w", err) + } + } + + output := formatters.MultipleRunOutput[runner.RunResult]{ + Runs: runsResults, + Resources: resources, + RunGroup: runGroup, + RunnerGetter: runnerGetter, + HasResults: true, + } + + summary := o.multipleRunFormatter.Format(output, formatters.Output(outputFormat)) + fmt.Println(summary) + + exitCode = ExitCodeSuccess + if runGroup.GetStatus() == "failed" { + exitCode = ExitCodeTestNotPassed + } + + return exitCode, nil +} + +func (o orchestrator) getDefinitionFiles(file []string) ([]string, error) { + files := make([]string, 0) + + for _, f := range file { + files = append(files, fileutil.ReadDirFileNames(f)...) + } + + return files, nil +} + +func (o orchestrator) createRun(ctx context.Context, resourceFetcher runner.ResourceFetcher, vars *varset.VarSets, requiredGates []string, definitionFile string, varsID string, runGroupID string) (runner.RunResult, any, error) { + resource, err := resourceFetcher.FetchWithDefinitionFile(ctx, definitionFile) + if err != nil { + return runner.RunResult{}, nil, err + } + + resourceType, err := resourcemanager.GetResourceType(resource) + if err != nil { + return runner.RunResult{}, nil, fmt.Errorf("cannot extract type from resource: %w", err) + } + + r, err := o.runnerRegistry.Get(resourceType) + if err != nil { + return runner.RunResult{}, nil, fmt.Errorf("cannot find runner for resource type %s: %w", resourceType, err) + } + + runInfo := openapi.RunInformation{ + VariableSetId: &varsID, + Variables: vars.ToOpenapi(), + Metadata: metadata.GetMetadata(), + RequiredGates: getRequiredGates(requiredGates), + RunGroupId: &runGroupID, + } + + // 2. validate missing vars + for { + result, err := r.StartRun(ctx, resource, runInfo) + if err == nil { + return result, resource, nil + } + if !errors.Is(err, varset.MissingVarsError{}) { + // actual error, return + return result, resource, fmt.Errorf("cannot run test: %w", err) + } + + // missing vars error + newVars := varset.AskForMissingVars([]varset.VarSet(err.(varset.MissingVarsError))) + vars = &newVars + o.logger.Debug("filled variables", zap.Any("variables", vars)) + } +} + +func (o orchestrator) waitForRunGroup(ctx context.Context, runGroupID string) (openapi.RunGroup, error) { + var ( + updatedResult openapi.RunGroup + lastError error + wg sync.WaitGroup + ) + + wg.Add(1) + ticker := time.NewTicker(1 * time.Second) // TODO: change to websockets + go func() { + for range ticker.C { + req := o.openapiClient.ApiApi.GetRunGroup(ctx, runGroupID) + runGroup, _, err := req.Execute() + + // updatedResult = runGroup + o.logger.Debug("updated run group", zap.String("result", spew.Sdump(runGroup))) + if err != nil { + o.logger.Debug("UpdateResult failed", zap.Error(err)) + lastError = err + wg.Done() + return + } + + if runGroup.GetStatus() == "succeed" || runGroup.GetStatus() == "failed" { + o.logger.Debug("result is finished") + updatedResult = *runGroup + wg.Done() + return + } + o.logger.Debug("still waiting") + } + }() + wg.Wait() + + if lastError != nil { + return openapi.RunGroup{}, lastError + } + + return updatedResult, nil +} + +var ErrJUnitNotSupported = errors.New("junit report is not supported for this resource type") + +func (a orchestrator) writeJUnitReport(ctx context.Context, r runner.Runner, result runner.RunResult, outputFile string) error { + if outputFile == "" { + a.logger.Debug("no junit output file specified") + return nil + } + + a.logger.Debug("saving junit report", zap.String("outputFile", outputFile)) + + report, err := r.JUnitResult(ctx, result) + if err != nil { + return err + } + + a.logger.Debug("junit report", zap.String("report", report)) + f, err := os.Create(outputFile) + if err != nil { + return fmt.Errorf("could not create junit output file: %w", err) + } + + _, err = f.WriteString(report) + + return err +} + +var source = "cli" + +func getRequiredGates(gates []string) []openapi.SupportedGates { + if len(gates) == 0 { + return nil + } + requiredGates := make([]openapi.SupportedGates, 0, len(gates)) + + for _, g := range gates { + requiredGates = append(requiredGates, openapi.SupportedGates(g)) + } + + return requiredGates +} + +// HandleRunError handles errors returned by the server when running a test. +// It normalizes the handling of general errors, like 404, +// but more importantly, it processes the missing environment variables error +// so the orchestrator can request them from the user. +func HandleRunError(resp *http.Response, reqErr error) error { + body, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("could not read response body: %w", err) + } + resp.Body.Close() + + if resp.StatusCode == http.StatusNotFound { + return fmt.Errorf("resource not found in server") + } + + if resp.StatusCode == http.StatusUnprocessableEntity { + return varset.BuildMissingVarsError(body) + } + + if reqErr != nil { + return fmt.Errorf("could not run test suite: %w", reqErr) + } + + return nil +} diff --git a/cli/cmd/config.go b/cli/cmd/config.go index ecebee3d5b..ca2cee1908 100644 --- a/cli/cmd/config.go +++ b/cli/cmd/config.go @@ -6,12 +6,12 @@ import ( "os" "github.com/kubeshop/tracetest/cli/analytics" + "github.com/kubeshop/tracetest/cli/cmdutil" "github.com/kubeshop/tracetest/cli/config" "github.com/kubeshop/tracetest/cli/formatters" "github.com/kubeshop/tracetest/cli/openapi" "github.com/spf13/cobra" "go.uber.org/zap" - "go.uber.org/zap/zapcore" ) var ( @@ -130,34 +130,7 @@ func validateConfig(cmd *cobra.Command, args []string) { } func setupLogger(cmd *cobra.Command, args []string) { - atom := zap.NewAtomicLevel() - if verbose { - atom.SetLevel(zap.DebugLevel) - } else { - atom.SetLevel(zap.WarnLevel) - } - - encoderCfg := zapcore.EncoderConfig{ - TimeKey: zapcore.OmitKey, - LevelKey: "level", - NameKey: zapcore.OmitKey, - CallerKey: zapcore.OmitKey, - FunctionKey: zapcore.OmitKey, - MessageKey: "message", - StacktraceKey: zapcore.OmitKey, - LineEnding: zapcore.DefaultLineEnding, - EncodeLevel: zapcore.CapitalColorLevelEncoder, - EncodeTime: zapcore.EpochTimeEncoder, - EncodeDuration: zapcore.SecondsDurationEncoder, - EncodeCaller: zapcore.ShortCallerEncoder, - } - - logger := zap.New(zapcore.NewCore( - zapcore.NewConsoleEncoder(encoderCfg), - zapcore.Lock(os.Stdout), - atom, - )) - *cliLogger = *logger + cliLogger = cmdutil.GetLogger(cmdutil.WithVerbose(verbose)) } func teardownCommand(cmd *cobra.Command, args []string) { diff --git a/cli/cmd/configure_cmd.go b/cli/cmd/configure_cmd.go index 877393a0f4..0ad4db749b 100644 --- a/cli/cmd/configure_cmd.go +++ b/cli/cmd/configure_cmd.go @@ -5,6 +5,7 @@ import ( "net/url" agentConfig "github.com/kubeshop/tracetest/agent/config" + "github.com/kubeshop/tracetest/cli/cmdutil" "github.com/kubeshop/tracetest/cli/config" "github.com/spf13/cobra" ) @@ -96,14 +97,14 @@ func (p *configureParameters) Validate(cmd *cobra.Command, args []string) []erro if flagUpdated { if p.ServerURL == "" { - errors = append(errors, paramError{ + errors = append(errors, cmdutil.ParamError{ Parameter: "server-url", Message: "server-url cannot be empty", }) } else { _, err := url.Parse(p.ServerURL) if err != nil { - errors = append(errors, paramError{ + errors = append(errors, cmdutil.ParamError{ Parameter: "server-url", Message: "server-url is not a valid URL", }) diff --git a/cli/cmd/legacy_test_cmd.go b/cli/cmd/legacy_test_cmd.go index adcefe46c9..48a002a63d 100644 --- a/cli/cmd/legacy_test_cmd.go +++ b/cli/cmd/legacy_test_cmd.go @@ -60,7 +60,7 @@ var testRunCmd = &cobra.Command{ PreRun: setupCommand(), Run: func(_ *cobra.Command, _ []string) { // map old flags to new ones - runParams.DefinitionFile = runTestFileDefinition + runParams.DefinitionFiles = []string{runTestFileDefinition} runParams.VarsID = runTestEnvID runParams.SkipResultWait = !runTestWaitForResult runParams.JUnitOuptutFile = runTestJUnit diff --git a/cli/cmd/middleware.go b/cli/cmd/middleware.go index 79b1b56e79..deebc37b32 100644 --- a/cli/cmd/middleware.go +++ b/cli/cmd/middleware.go @@ -6,6 +6,7 @@ import ( "fmt" "os" + "github.com/kubeshop/tracetest/cli/cmdutil" "github.com/kubeshop/tracetest/cli/config" "github.com/kubeshop/tracetest/cli/pkg/resourcemanager" "github.com/kubeshop/tracetest/cli/ui" @@ -116,7 +117,7 @@ func handleErrorMessage(err error) string { return fmt.Sprintf("user is not authenticated on %s", cliConfig.Endpoint) } -func WithParamsHandler(validators ...Validator) MiddlewareWrapper { +func WithParamsHandler(validators ...cmdutil.Validator) MiddlewareWrapper { return func(runFn RunFn) RunFn { return func(ctx context.Context, cmd *cobra.Command, args []string) (string, error) { errors := make([]error, 0) @@ -139,11 +140,7 @@ func WithParamsHandler(validators ...Validator) MiddlewareWrapper { } } -type Validator interface { - Validate(cmd *cobra.Command, args []string) []error -} - -func WithResourceMiddleware(runFn RunFn, params ...Validator) CobraRunFn { +func WithResourceMiddleware(runFn RunFn, params ...cmdutil.Validator) CobraRunFn { params = append(params, resourceParams) return WithResultHandler(WithParamsHandler(params...)(runFn)) } @@ -166,7 +163,7 @@ func (p *resourceParameters) Validate(cmd *cobra.Command, args []string) []error if len(args) == 0 || args[0] == "" { return []error{ - paramError{ + cmdutil.ParamError{ Parameter: "resource", Message: fmt.Sprintf("resource name must be provided. Available resources: %s", resourceList()), }, @@ -180,7 +177,7 @@ func (p *resourceParameters) Validate(cmd *cobra.Command, args []string) []error suggestion := resources.Suggest(p.ResourceName) if suggestion != "" { return []error{ - paramError{ + cmdutil.ParamError{ Parameter: "resource", Message: fmt.Sprintf("resource \"%s\" not found. Did you mean this?\n\t%s", p.ResourceName, suggestion), }, @@ -188,7 +185,7 @@ func (p *resourceParameters) Validate(cmd *cobra.Command, args []string) []error } return []error{ - paramError{ + cmdutil.ParamError{ Parameter: "resource", Message: fmt.Sprintf("resource must be %s", resourceList()), }, @@ -197,12 +194,3 @@ func (p *resourceParameters) Validate(cmd *cobra.Command, args []string) []error return nil } - -type paramError struct { - Parameter string - Message string -} - -func (pe paramError) Error() string { - return fmt.Sprintf(`[%s] %s`, pe.Parameter, pe.Message) -} diff --git a/cli/cmd/resource_apply_cmd.go b/cli/cmd/resource_apply_cmd.go index 2f9e902499..09aa12a13a 100644 --- a/cli/cmd/resource_apply_cmd.go +++ b/cli/cmd/resource_apply_cmd.go @@ -4,6 +4,7 @@ import ( "context" "fmt" + "github.com/kubeshop/tracetest/cli/cmdutil" "github.com/kubeshop/tracetest/cli/pkg/fileutil" "github.com/kubeshop/tracetest/cli/pkg/resourcemanager" "github.com/spf13/cobra" @@ -61,7 +62,7 @@ func (p applyParameters) Validate(cmd *cobra.Command, args []string) []error { errors := make([]error, 0) if p.DefinitionFile == "" { - errors = append(errors, paramError{ + errors = append(errors, cmdutil.ParamError{ Parameter: "file", Message: "Definition file must be provided", }) diff --git a/cli/cmd/resource_export_cmd.go b/cli/cmd/resource_export_cmd.go index a9d33cefeb..3b9b603cf4 100644 --- a/cli/cmd/resource_export_cmd.go +++ b/cli/cmd/resource_export_cmd.go @@ -5,6 +5,7 @@ import ( "fmt" "os" + "github.com/kubeshop/tracetest/cli/cmdutil" "github.com/kubeshop/tracetest/cli/pkg/resourcemanager" "github.com/spf13/cobra" ) @@ -59,7 +60,7 @@ func (p exportParameters) Validate(cmd *cobra.Command, args []string) []error { errors := p.resourceIDParameters.Validate(cmd, args) if p.OutputFile == "" { - errors = append(errors, paramError{ + errors = append(errors, cmdutil.ParamError{ Parameter: "file", Message: "output file must be provided", }) diff --git a/cli/cmd/resource_get_cmd.go b/cli/cmd/resource_get_cmd.go index 09760770f4..7362689859 100644 --- a/cli/cmd/resource_get_cmd.go +++ b/cli/cmd/resource_get_cmd.go @@ -4,6 +4,7 @@ import ( "context" "errors" + "github.com/kubeshop/tracetest/cli/cmdutil" "github.com/kubeshop/tracetest/cli/pkg/resourcemanager" "github.com/spf13/cobra" ) @@ -58,7 +59,7 @@ func (p resourceIDParameters) Validate(cmd *cobra.Command, args []string) []erro errors := make([]error, 0) if p.ResourceID == "" { - errors = append(errors, paramError{ + errors = append(errors, cmdutil.ParamError{ Parameter: "id", Message: "resource id must be provided", }) diff --git a/cli/cmd/resource_list_cmd.go b/cli/cmd/resource_list_cmd.go index a2536b36b2..a449a1172c 100644 --- a/cli/cmd/resource_list_cmd.go +++ b/cli/cmd/resource_list_cmd.go @@ -3,6 +3,7 @@ package cmd import ( "context" + "github.com/kubeshop/tracetest/cli/cmdutil" "github.com/kubeshop/tracetest/cli/pkg/resourcemanager" "github.com/spf13/cobra" ) @@ -71,21 +72,21 @@ func (p listParameters) Validate(cmd *cobra.Command, args []string) []error { errors := make([]error, 0) if p.Take < 0 { - errors = append(errors, paramError{ + errors = append(errors, cmdutil.ParamError{ Parameter: "take", Message: "take parameter must be greater than 0", }) } if p.Skip < 0 { - errors = append(errors, paramError{ + errors = append(errors, cmdutil.ParamError{ Parameter: "skip", Message: "skip parameter must be greater than 0", }) } if p.SortDirection != "" && p.SortDirection != "asc" && p.SortDirection != "desc" { - errors = append(errors, paramError{ + errors = append(errors, cmdutil.ParamError{ Parameter: "sortDirection", Message: "sortDirection parameter must be either asc or desc", }) diff --git a/cli/cmd/resource_run_cmd.go b/cli/cmd/resource_run_cmd.go index 1eb2022b4a..1afd33554e 100644 --- a/cli/cmd/resource_run_cmd.go +++ b/cli/cmd/resource_run_cmd.go @@ -5,75 +5,48 @@ import ( "fmt" "strings" + "github.com/kubeshop/tracetest/cli/cmdutil" "github.com/kubeshop/tracetest/cli/config" "github.com/kubeshop/tracetest/cli/openapi" - "github.com/kubeshop/tracetest/cli/pkg/fileutil" "github.com/kubeshop/tracetest/cli/runner" "github.com/spf13/cobra" + + cloudCmd "github.com/kubeshop/tracetest/cli/cloud/cmd" ) var ( - runParams = &runParameters{} + runParams = &cmdutil.RunParameters{} runCmd *cobra.Command ) func init() { runCmd = &cobra.Command{ GroupID: cmdGroupResources.ID, - Use: "run " + runnableResourceList(), - Short: "run resources", - Long: "run resources", + Use: fmt.Sprintf("run [%s]", runnableResourceList()), + Short: "Run tests and test suites", + Long: "Run tests and test suites", PreRun: setupCommand(WithOptionalResourceName()), Run: WithResourceMiddleware(func(ctx context.Context, _ *cobra.Command, args []string) (string, error) { - resourceType, err := getResourceType(runParams, resourceParams) - if err != nil { - return "", err - } + if cliConfig.Jwt != "" { + exitCode, err := cloudCmd.RunMultipleFiles(ctx, runParams, &cliConfig, runnerRegistry, output) - r, err := runnerRegistry.Get(resourceType) - if err != nil { - return "", fmt.Errorf("resource type '%s' cannot be run", resourceType) - } - - orchestrator := runner.Orchestrator( - cliLogger, - config.GetAPIClient(cliConfig), - variableSetClient, - ) - - if runParams.EnvID != "" { - runParams.VarsID = runParams.EnvID - } - - runParams := runner.RunOptions{ - ID: runParams.ID, - DefinitionFile: runParams.DefinitionFile, - VarsID: runParams.VarsID, - SkipResultWait: runParams.SkipResultWait, - JUnitOuptutFile: runParams.JUnitOuptutFile, - RequiredGates: runParams.RequriedGates, - } - - exitCode, err := orchestrator.Run(ctx, r, runParams, output) - if err != nil { + ExitCLI(exitCode) return "", err } - ExitCLI(exitCode) - - // ExitCLI will exit the process, so this return is just to satisfy the compiler - return "", nil + return runSingleFile(ctx) }, runParams), PostRun: teardownCommand, } - runCmd.Flags().StringVarP(&runParams.DefinitionFile, "file", "f", "", "path to the definition file") - runCmd.Flags().StringVar(&runParams.ID, "id", "", "id of the resource to run") + runCmd.Flags().StringSliceVarP(&runParams.DefinitionFiles, "file", "f", []string{}, "path to the definition file (can be defined multiple times)") + runCmd.Flags().StringVarP(&runParams.ID, "id", "", "", "id of the resource to run (can be defined multiple times)") runCmd.Flags().StringVarP(&runParams.VarsID, "vars", "", "", "variable set file or ID to be used") + runCmd.Flags().StringVarP(&runParams.RunGroupID, "group", "g", "", "Sets the Run Group ID for the run. This is used to group multiple runs together.") runCmd.Flags().BoolVarP(&runParams.SkipResultWait, "skip-result-wait", "W", false, "do not wait for results. exit immediately after test run started") runCmd.Flags().StringVarP(&runParams.JUnitOuptutFile, "junit", "j", "", "file path to save test results in junit format") - runCmd.Flags().StringSliceVar(&runParams.RequriedGates, "required-gates", []string{}, "override default required gate. "+validRequiredGatesMsg()) + runCmd.Flags().StringSliceVar(&runParams.RequiredGates, "required-gates", []string{}, "override default required gate. "+validRequiredGatesMsg()) //deprecated runCmd.Flags().StringVarP(&runParams.EnvID, "environment", "e", "", "environment file or ID to be used") @@ -83,23 +56,37 @@ func init() { rootCmd.AddCommand(runCmd) } -func getResourceType(runParams *runParameters, resourceParams *resourceParameters) (string, error) { - if resourceParams.ResourceName != "" { - return resourceParams.ResourceName, nil +func runSingleFile(ctx context.Context) (string, error) { + orchestrator := runner.Orchestrator( + cliLogger, + config.GetAPIClient(cliConfig), + variableSetClient, + runnerRegistry, + ) + + if runParams.EnvID != "" { + runParams.VarsID = runParams.EnvID } - if runParams.DefinitionFile != "" { - filePath := runParams.DefinitionFile - f, err := fileutil.Read(filePath) - if err != nil { - return "", fmt.Errorf("cannot read file %s: %w", filePath, err) - } + var definitionFile string + if len(runParams.DefinitionFiles) > 0 { + definitionFile = runParams.DefinitionFiles[0] + } - return strings.ToLower(f.Type()), nil + runParams := runner.RunOptions{ + ID: runParams.ID, + DefinitionFile: definitionFile, + VarsID: runParams.VarsID, + SkipResultWait: runParams.SkipResultWait, + JUnitOuptutFile: runParams.JUnitOuptutFile, + RequiredGates: runParams.RequiredGates, } - return "", fmt.Errorf("resourceName is empty and no definition file provided") + exitCode, err := orchestrator.Run(ctx, runParams, output) + ExitCLI(exitCode) + // ExitCLI will exit the process, so this return is just to satisfy the compiler + return "", err } func validRequiredGatesMsg() string { @@ -110,49 +97,3 @@ func validRequiredGatesMsg() string { return "valid options: " + strings.Join(opts, ", ") } - -type runParameters struct { - ID string - DefinitionFile string - VarsID string - EnvID string - SkipResultWait bool - JUnitOuptutFile string - RequriedGates []string -} - -func (p runParameters) Validate(cmd *cobra.Command, args []string) []error { - errs := []error{} - if p.DefinitionFile == "" && p.ID == "" { - errs = append(errs, paramError{ - Parameter: "resource", - Message: "you must specify a definition file or resource ID", - }) - } - - if p.DefinitionFile != "" && p.ID != "" { - errs = append(errs, paramError{ - Parameter: "resource", - Message: "you cannot specify both a definition file and resource ID", - }) - } - - if p.JUnitOuptutFile != "" && p.SkipResultWait { - errs = append(errs, paramError{ - Parameter: "junit", - Message: "--junit option is incompatible with --skip-result-wait option", - }) - } - - for _, rg := range p.RequriedGates { - _, err := openapi.NewSupportedGatesFromValue(rg) - if err != nil { - errs = append(errs, paramError{ - Parameter: "required-gates", - Message: fmt.Sprintf("invalid option '%s'. "+validRequiredGatesMsg(), rg), - }) - } - } - - return errs -} diff --git a/cli/cmd/resources.go b/cli/cmd/resources.go index 37f7e4cee0..ec1efd0e91 100644 --- a/cli/cmd/resources.go +++ b/cli/cmd/resources.go @@ -9,6 +9,7 @@ import ( "github.com/Jeffail/gabs/v2" "github.com/kubeshop/tracetest/cli/analytics" + "github.com/kubeshop/tracetest/cli/cmdutil" "github.com/kubeshop/tracetest/cli/formatters" "github.com/kubeshop/tracetest/cli/pkg/fileutil" "github.com/kubeshop/tracetest/cli/pkg/resourcemanager" @@ -39,20 +40,7 @@ var ( httpClient = &resourcemanager.HTTPClient{} variableSetPreprocessor = preprocessor.VariableSet(cliLogger) - variableSetClient = resourcemanager.NewClient( - httpClient, cliLogger, - "variableset", "variablesets", - resourcemanager.WithTableConfig(resourcemanager.TableConfig{ - Cells: []resourcemanager.TableCellConfig{ - {Header: "ID", Path: "spec.id"}, - {Header: "NAME", Path: "spec.name"}, - {Header: "DESCRIPTION", Path: "spec.description"}, - }, - }), - resourcemanager.WithResourceType("VariableSet"), - resourcemanager.WithApplyPreProcessor(variableSetPreprocessor.Preprocess), - resourcemanager.WithDeprecatedAlias("Environment"), - ) + variableSetClient = GetVariableSetClient(httpClient, variableSetPreprocessor) testPreprocessor = preprocessor.Test(cliLogger) testClient = resourcemanager.NewClient( @@ -336,3 +324,22 @@ func formatItemDate(item *gabs.Container, path string) error { item.SetP(date.Format(time.DateTime), path) return nil } + +func GetVariableSetClient(httpClient *resourcemanager.HTTPClient, preprocessor preprocessor.Preprocessor) resourcemanager.Client { + variableSetClient := resourcemanager.NewClient( + httpClient, cmdutil.GetLogger(), + "variableset", "variablesets", + resourcemanager.WithTableConfig(resourcemanager.TableConfig{ + Cells: []resourcemanager.TableCellConfig{ + {Header: "ID", Path: "spec.id"}, + {Header: "NAME", Path: "spec.name"}, + {Header: "DESCRIPTION", Path: "spec.description"}, + }, + }), + resourcemanager.WithResourceType("VariableSet"), + resourcemanager.WithApplyPreProcessor(preprocessor.Preprocess), + resourcemanager.WithDeprecatedAlias("Environment"), + ) + + return variableSetClient +} diff --git a/cli/cmd/server_install_cmd.go b/cli/cmd/server_install_cmd.go index 3aaaf82868..578e52ea3c 100644 --- a/cli/cmd/server_install_cmd.go +++ b/cli/cmd/server_install_cmd.go @@ -1,6 +1,7 @@ package cmd import ( + "github.com/kubeshop/tracetest/cli/cmdutil" "github.com/kubeshop/tracetest/cli/installer" "github.com/spf13/cobra" "golang.org/x/exp/slices" @@ -60,25 +61,25 @@ type installerParameters struct { KubernetesContext string } -func (p installerParameters) Validate(cmd *cobra.Command, args []string) []paramError { - errors := make([]paramError, 0) +func (p installerParameters) Validate(cmd *cobra.Command, args []string) []cmdutil.ParamError { + errors := make([]cmdutil.ParamError, 0) if cmd.Flags().Lookup("run-environment").Changed && slices.Contains(AllowedRunEnvironments, p.RunEnvironment) { - errors = append(errors, paramError{ + errors = append(errors, cmdutil.ParamError{ Parameter: "run-environment", Message: "run-environment must be one of 'none', 'docker' or 'kubernetes'", }) } if cmd.Flags().Lookup("mode").Changed && slices.Contains(AllowedInstallationMode, p.InstallationMode) { - errors = append(errors, paramError{ + errors = append(errors, cmdutil.ParamError{ Parameter: "mode", Message: "mode must be one of 'not-chosen', 'with-demo' or 'just-tracetest'", }) } if cmd.Flags().Lookup("kubernetes-context").Changed && p.KubernetesContext == "" { - errors = append(errors, paramError{ + errors = append(errors, cmdutil.ParamError{ Parameter: "kubernetes-context", Message: "kubernetes-context cannot be empty", }) diff --git a/cli/cmdutil/common_params.go b/cli/cmdutil/common_params.go new file mode 100644 index 0000000000..c31ddb3007 --- /dev/null +++ b/cli/cmdutil/common_params.go @@ -0,0 +1,20 @@ +package cmdutil + +import ( + "fmt" + + "github.com/spf13/cobra" +) + +type Validator interface { + Validate(cmd *cobra.Command, args []string) []error +} + +type ParamError struct { + Parameter string + Message string +} + +func (pe ParamError) Error() string { + return fmt.Sprintf(`[%s] %s`, pe.Parameter, pe.Message) +} diff --git a/cli/cmdutil/logger.go b/cli/cmdutil/logger.go new file mode 100644 index 0000000000..69b0a04916 --- /dev/null +++ b/cli/cmdutil/logger.go @@ -0,0 +1,61 @@ +package cmdutil + +import ( + "os" + + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +var logger *zap.Logger + +type loggerConfig struct { + Verbose bool +} + +type loggerOption func(*loggerConfig) + +func WithVerbose(verbose bool) loggerOption { + return func(c *loggerConfig) { + c.Verbose = verbose + } +} + +func GetLogger(opts ...loggerOption) *zap.Logger { + if logger != nil { + return logger + } + + loggerConfig := loggerConfig{} + for _, opt := range opts { + opt(&loggerConfig) + } + + atom := zap.NewAtomicLevel() + if loggerConfig.Verbose { + atom.SetLevel(zap.DebugLevel) + } else { + atom.SetLevel(zap.WarnLevel) + } + + encoderCfg := zapcore.EncoderConfig{ + TimeKey: zapcore.OmitKey, + LevelKey: "level", + NameKey: zapcore.OmitKey, + CallerKey: zapcore.OmitKey, + FunctionKey: zapcore.OmitKey, + MessageKey: "message", + StacktraceKey: zapcore.OmitKey, + LineEnding: zapcore.DefaultLineEnding, + EncodeLevel: zapcore.CapitalColorLevelEncoder, + EncodeTime: zapcore.EpochTimeEncoder, + EncodeDuration: zapcore.SecondsDurationEncoder, + EncodeCaller: zapcore.ShortCallerEncoder, + } + + return zap.New(zapcore.NewCore( + zapcore.NewConsoleEncoder(encoderCfg), + zapcore.Lock(os.Stdout), + atom, + )) +} diff --git a/cli/cmdutil/resources.go b/cli/cmdutil/resources.go new file mode 100644 index 0000000000..df0119b8ff --- /dev/null +++ b/cli/cmdutil/resources.go @@ -0,0 +1,17 @@ +package cmdutil + +import ( + "fmt" + "strings" + + "github.com/kubeshop/tracetest/cli/pkg/fileutil" +) + +func GetResourceTypeFromFile(filePath string) (string, error) { + f, err := fileutil.Read(filePath) + if err != nil { + return "", fmt.Errorf("cannot read file %s: %w", filePath, err) + } + + return strings.ToLower(f.Type()), nil +} diff --git a/cli/cmdutil/run_params.go b/cli/cmdutil/run_params.go new file mode 100644 index 0000000000..e09c30879b --- /dev/null +++ b/cli/cmdutil/run_params.go @@ -0,0 +1,69 @@ +package cmdutil + +import ( + "fmt" + "strings" + + "github.com/kubeshop/tracetest/cli/openapi" + "github.com/spf13/cobra" +) + +type RunParameters struct { + ID string + DefinitionFiles []string + VarsID string + EnvID string + SkipResultWait bool + JUnitOuptutFile string + RequiredGates []string + RunGroupID string +} + +func (p RunParameters) Validate(cmd *cobra.Command, args []string) []error { + errs := []error{} + + hasDefinitionFilesSpecified := p.DefinitionFiles != nil && len(p.DefinitionFiles) > 0 + hasFileIDsSpecified := p.ID != "" && len(p.ID) > 0 + + if !hasDefinitionFilesSpecified && !hasFileIDsSpecified { + errs = append(errs, ParamError{ + Parameter: "resource", + Message: "you must specify at least one definition file or resource ID", + }) + } + + if hasDefinitionFilesSpecified && hasFileIDsSpecified { + errs = append(errs, ParamError{ + Parameter: "resource", + Message: "you cannot specify both a definition file and resource ID", + }) + } + + if p.JUnitOuptutFile != "" && p.SkipResultWait { + errs = append(errs, ParamError{ + Parameter: "junit", + Message: "--junit option is incompatible with --skip-result-wait option", + }) + } + + for _, rg := range p.RequiredGates { + _, err := openapi.NewSupportedGatesFromValue(rg) + if err != nil { + errs = append(errs, ParamError{ + Parameter: "required-gates", + Message: fmt.Sprintf("invalid option '%s'. %s", rg, validRequiredGatesMsg()), + }) + } + } + + return errs +} + +func validRequiredGatesMsg() string { + opts := make([]string, 0, len(openapi.AllowedSupportedGatesEnumValues)) + for _, v := range openapi.AllowedSupportedGatesEnumValues { + opts = append(opts, string(v)) + } + + return "valid options: " + strings.Join(opts, ", ") +} diff --git a/cli/formatters/multiple_runs.go b/cli/formatters/multiple_runs.go new file mode 100644 index 0000000000..afd58c1b32 --- /dev/null +++ b/cli/formatters/multiple_runs.go @@ -0,0 +1,121 @@ +package formatters + +import ( + "encoding/json" + "fmt" + "strings" + + "github.com/kubeshop/tracetest/cli/openapi" + "github.com/pterm/pterm" +) + +type RunnerGetter[T any] func(resource any) (Runner[T], error) + +type Runner[T any] interface { + FormatResult(resource T, format string) string +} + +type multipleRun[T any] struct { + baseURLFn func() string + colorsEnabled bool +} + +func MultipleRun[T any](baseURLFn func() string, colorsEnabled bool) multipleRun[T] { + return multipleRun[T]{ + baseURLFn: baseURLFn, + colorsEnabled: colorsEnabled, + } +} + +type MultipleRunOutput[T any] struct { + Runs []T + Resources []any + HasResults bool + RunGroup openapi.RunGroup + RunnerGetter RunnerGetter[T] +} + +func (f multipleRun[T]) Format(output MultipleRunOutput[T], format Output) string { + switch format { + case JSON: + return f.json(output) + case Pretty, "": + return f.pretty(output) + } + + return "" +} + +type jsonSummary struct { + RunGroup openapi.RunGroup `json:"runGroup"` + RunGroupUrl string `json:"runGroupUrl"` + Runs []any `json:"runs"` +} + +func (f multipleRun[T]) json(output MultipleRunOutput[T]) string { + summary := jsonSummary{ + RunGroup: output.RunGroup, + RunGroupUrl: fmt.Sprintf("%s/run/%s", f.baseURLFn(), output.RunGroup.GetId()), + Runs: make([]any, 0, len(output.Runs)), + } + + for i, run := range output.Runs { + resource := output.Resources[i] + runner, _ := output.RunnerGetter(resource) + result := runner.FormatResult(run, "json") + + var output any + + json.Unmarshal([]byte(result), &output) + + summary.Runs = append(summary.Runs, output) + } + + bytes, err := json.MarshalIndent(summary, "", " ") + if err != nil { + panic(fmt.Errorf("could not marshal output json: %w", err)) + } + + return string(bytes) + "\n" +} + +var messageTemplate = "%s RunGroup: #%s (%s)\n" + +func (f multipleRun[T]) pretty(output MultipleRunOutput[T]) string { + runGroupUrl := fmt.Sprintf("%s/run/%s", f.baseURLFn(), output.RunGroup.GetId()) + if !output.HasResults { + return fmt.Sprintf(messageTemplate, PROGRESS_TEST_ICON, output.RunGroup.GetId(), runGroupUrl) + } + + message := fmt.Sprintf(messageTemplate, PASSED_TEST_ICON, output.RunGroup.GetId(), runGroupUrl) + allStepsPassed := output.RunGroup.Summary.GetFailed() == 0 + if !allStepsPassed { + message = fmt.Sprintf(messageTemplate, FAILED_TEST_ICON, output.RunGroup.GetId(), runGroupUrl) + } + + // the test suite name + all steps + formattedMessages := make([]string, 0, len(output.Runs)+1) + formattedMessages = append(formattedMessages, f.getColoredText(allStepsPassed, message)) + + for i, run := range output.Runs { + resource := output.Resources[i] + runner, _ := output.RunnerGetter(resource) + result := runner.FormatResult(run, "pretty") + + formattedMessages = append(formattedMessages, result) + } + + return strings.Join(formattedMessages, " ") +} + +func (f multipleRun[T]) getColoredText(passed bool, text string) string { + if !f.colorsEnabled { + return text + } + + if passed { + return pterm.FgGreen.Sprintf(text) + } + + return pterm.FgRed.Sprintf(text) +} diff --git a/cli/formatters/test_run.go b/cli/formatters/test_run.go index 24f0abd869..f0523a5d7b 100644 --- a/cli/formatters/test_run.go +++ b/cli/formatters/test_run.go @@ -11,8 +11,9 @@ import ( ) const ( - PASSED_TEST_ICON = "✔" - FAILED_TEST_ICON = "✘" + PASSED_TEST_ICON = "✔" + FAILED_TEST_ICON = "✘" + PROGRESS_TEST_ICON = "🚀" ) type testRun struct { diff --git a/cli/metadata/metadata.go b/cli/metadata/metadata.go new file mode 100644 index 0000000000..498f96a5e1 --- /dev/null +++ b/cli/metadata/metadata.go @@ -0,0 +1,54 @@ +package metadata + +import ( + cienvironment "github.com/cucumber/ci-environment/go" + "github.com/kubeshop/tracetest/cli/config" +) + +var ( + tracetestSource = "tracetest.source" + tracetestCliVersion = "tracetest.cli.version" + gitRemote = "git.GitRemote" + gitBranch = "git.branch" + gitTag = "git.tag" + gitSha = "git.sha" + cIBuildNumber = "ci.build.number" + cIProvider = "ci.provider" + cIBuildUrl = "ci.build.url" +) + +type Metadata map[string]string + +func (m Metadata) Merge(other Metadata) Metadata { + for k, v := range other { + m[k] = v + } + + return m +} + +func GetMetadata() Metadata { + // TODO: add more metadata after getting the response from the k6 team + // https://github.com/grafana/k6/issues/1320#issuecomment-2032734378 + metadata := Metadata{} + metadata[tracetestSource] = "cli" + metadata[tracetestCliVersion] = config.Version + + ci := cienvironment.DetectCIEnvironment() + if ci == nil { + return metadata + } + + metadata[cIProvider] = ci.Name + metadata[cIBuildUrl] = ci.URL + metadata[cIBuildNumber] = ci.BuildNumber + metadata[gitBranch] = ci.Git.Branch + + if ci.Git != nil { + metadata[gitTag] = ci.Git.Tag + metadata[gitSha] = ci.Git.Revision + metadata[gitRemote] = ci.Git.Remote + } + + return metadata +} diff --git a/cli/pkg/fileutil/file.go b/cli/pkg/fileutil/file.go index d912959e82..7cb77611ca 100644 --- a/cli/pkg/fileutil/file.go +++ b/cli/pkg/fileutil/file.go @@ -16,6 +16,43 @@ type File struct { contents []byte } +func IsDir(path string) bool { + file, err := os.Stat(path) + if err != nil { + return false + } + + return file.IsDir() +} + +func ReadDirFileNames(path string) []string { + path, err := filepath.Abs(path) + if err != nil { + return []string{} + } + + if !IsDir(path) { + return []string{path} + } + + files, err := os.ReadDir(path) + if err != nil { + return []string{} + } + + var result []string + for _, file := range files { + // TODO: add validation for file extensions, tracetest runnable definitions (?) + if file.IsDir() { + result = append(result, ReadDirFileNames(filepath.Join(path, file.Name()))...) + } else { + result = append(result, filepath.Join(path, file.Name())) + } + } + + return result +} + func Read(filePath string) (File, error) { b, err := os.ReadFile(filePath) if err != nil { diff --git a/cli/pkg/resourcemanager/client.go b/cli/pkg/resourcemanager/client.go index 5e66d42d81..56dd0842ff 100644 --- a/cli/pkg/resourcemanager/client.go +++ b/cli/pkg/resourcemanager/client.go @@ -9,6 +9,7 @@ import ( "path" "strings" + "github.com/kubeshop/tracetest/cli/cmdutil" "go.uber.org/zap" "golang.org/x/text/cases" "golang.org/x/text/language" @@ -95,7 +96,7 @@ func NewClient( client: httpClient, resourceName: resourceName, resourceNamePlural: resourceNamePlural, - logger: logger, + logger: cmdutil.GetLogger(), } for _, opt := range opts { diff --git a/cli/pkg/resourcemanager/util.go b/cli/pkg/resourcemanager/util.go new file mode 100644 index 0000000000..ac0647e962 --- /dev/null +++ b/cli/pkg/resourcemanager/util.go @@ -0,0 +1,28 @@ +package resourcemanager + +import ( + "fmt" + "reflect" +) + +func GetResourceType(input any) (string, error) { + value := reflect.ValueOf(input) + if value.Kind() != reflect.Struct { + return "", fmt.Errorf("input must be a struct") + } + + for i := 0; i < value.NumField(); i++ { + valueField := value.Field(i) + typeField := value.Type().Field(i) + + if typeField.Name == "Type" { + f := valueField.Interface() + val := reflect.ValueOf(f) + val = reflect.Indirect(val) + + return val.String(), nil + } + } + + return "", fmt.Errorf(`struct has no "Type" field`) +} diff --git a/cli/pkg/resourcemanager/util_test.go b/cli/pkg/resourcemanager/util_test.go new file mode 100644 index 0000000000..a647c7a47e --- /dev/null +++ b/cli/pkg/resourcemanager/util_test.go @@ -0,0 +1,55 @@ +package resourcemanager_test + +import ( + "testing" + + "github.com/kubeshop/tracetest/cli/pkg/resourcemanager" + "gotest.tools/v3/assert" +) + +func TestGetResourceType(t *testing.T) { + type resource struct { + Type string + Spec any + } + testCases := []struct { + Name string + Input any + ExpectedType string + ExpectedError string + }{ + { + Name: "should get correct `Test` type", + Input: resource{Type: "Test", Spec: "anything"}, + ExpectedType: "Test", + }, + { + Name: "should get correct `TestSuite` type", + Input: resource{Type: "TestSuite", Spec: "anything"}, + ExpectedType: "TestSuite", + }, + { + Name: "should fail when input is not a struct", + Input: "anything", + ExpectedError: "input must be a struct", + }, + { + Name: "should fail when input is an struct without Type field", + Input: struct{ Value string }{Value: "anything"}, + ExpectedError: `struct has no "Type" field`, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.Name, func(t *testing.T) { + resourceType, err := resourcemanager.GetResourceType(testCase.Input) + if testCase.ExpectedError != "" { + assert.Error(t, err, testCase.ExpectedError) + } + + if testCase.ExpectedType != "" { + assert.Equal(t, testCase.ExpectedType, resourceType) + } + }) + } +} diff --git a/cli/preprocessor/preprocessor.go b/cli/preprocessor/preprocessor.go new file mode 100644 index 0000000000..eb2b89751a --- /dev/null +++ b/cli/preprocessor/preprocessor.go @@ -0,0 +1,11 @@ +package preprocessor + +import ( + "context" + + "github.com/kubeshop/tracetest/cli/pkg/fileutil" +) + +type Preprocessor interface { + Preprocess(ctx context.Context, input fileutil.File) (fileutil.File, error) +} diff --git a/cli/preprocessor/test.go b/cli/preprocessor/test.go index 915c844be7..c243d39888 100644 --- a/cli/preprocessor/test.go +++ b/cli/preprocessor/test.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/goccy/go-yaml" + "github.com/kubeshop/tracetest/cli/cmdutil" "github.com/kubeshop/tracetest/cli/openapi" "github.com/kubeshop/tracetest/cli/pkg/fileutil" "go.uber.org/zap" @@ -16,7 +17,7 @@ type test struct { func Test(logger *zap.Logger) test { return test{ - logger: logger, + logger: cmdutil.GetLogger(), } } diff --git a/cli/preprocessor/testsuite.go b/cli/preprocessor/testsuite.go index 55437c4627..67d067a709 100644 --- a/cli/preprocessor/testsuite.go +++ b/cli/preprocessor/testsuite.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/goccy/go-yaml" + "github.com/kubeshop/tracetest/cli/cmdutil" "github.com/kubeshop/tracetest/cli/openapi" "github.com/kubeshop/tracetest/cli/pkg/fileutil" "go.uber.org/zap" @@ -19,7 +20,7 @@ type testSuite struct { func TestSuite(logger *zap.Logger, applyTestFn applyTestFunc) testSuite { return testSuite{ - logger: logger, + logger: cmdutil.GetLogger(), applyTestFn: applyTestFn, } } diff --git a/cli/preprocessor/variableset.go b/cli/preprocessor/variableset.go index 8d0a6959a6..cd2e03371b 100644 --- a/cli/preprocessor/variableset.go +++ b/cli/preprocessor/variableset.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/goccy/go-yaml" + "github.com/kubeshop/tracetest/cli/cmdutil" "github.com/kubeshop/tracetest/cli/pkg/fileutil" "go.uber.org/zap" ) @@ -20,7 +21,7 @@ type generic struct { func VariableSet(logger *zap.Logger) variableSet { return variableSet{ - logger: logger, + logger: cmdutil.GetLogger(), } } diff --git a/cli/runner/orchestrator.go b/cli/runner/orchestrator.go index 210f66f05d..220189bbd3 100644 --- a/cli/runner/orchestrator.go +++ b/cli/runner/orchestrator.go @@ -10,12 +10,13 @@ import ( "sync" "time" - cienvironment "github.com/cucumber/ci-environment/go" "github.com/davecgh/go-spew/spew" + "github.com/kubeshop/tracetest/cli/metadata" "github.com/kubeshop/tracetest/cli/openapi" "github.com/kubeshop/tracetest/cli/pkg/fileutil" "github.com/kubeshop/tracetest/cli/pkg/resourcemanager" "github.com/kubeshop/tracetest/cli/variable" + "github.com/kubeshop/tracetest/cli/varset" "go.uber.org/zap" "gopkg.in/yaml.v2" ) @@ -43,6 +44,11 @@ type RunOptions struct { // Overrides the default required gates for the resource RequiredGates []string + + // ResourceType defines what is the type of resource that is being run. It's value + // is filled automatically when the user define the type of resource that will be run + // when they enter: tracetest run --id + ResourceType string } // RunResult holds the result of the run @@ -90,19 +96,22 @@ func Orchestrator( logger *zap.Logger, openapiClient *openapi.APIClient, variableSets resourcemanager.Client, + runnerRegistry Registry, ) orchestrator { return orchestrator{ - logger: logger, - openapiClient: openapiClient, - variableSets: variableSets, + logger: logger, + openapiClient: openapiClient, + variableSets: variableSets, + runnerRegistry: runnerRegistry, } } type orchestrator struct { logger *zap.Logger - openapiClient *openapi.APIClient - variableSets resourcemanager.Client + openapiClient *openapi.APIClient + variableSets resourcemanager.Client + runnerRegistry Registry } var ( @@ -116,7 +125,7 @@ const ( ExitCodeTestNotPassed = 2 ) -func (o orchestrator) Run(ctx context.Context, r Runner, opts RunOptions, outputFormat string) (exitCode int, _ error) { +func (o orchestrator) Run(ctx context.Context, opts RunOptions, outputFormat string) (exitCode int, _ error) { o.logger.Debug( "Running test from definition", zap.String("definitionFile", opts.DefinitionFile), @@ -127,80 +136,76 @@ func (o orchestrator) Run(ctx context.Context, r Runner, opts RunOptions, output zap.Strings("requiredGates", opts.RequiredGates), ) - varsID, err := o.resolveVarsID(ctx, opts.VarsID) + variableSetFetcher := GetVariableSetFetcher(o.logger, o.variableSets) + + varsID, err := variableSetFetcher.Fetch(ctx, opts.VarsID) if err != nil { return ExitCodeGeneralError, fmt.Errorf("cannot resolve variable set id: %w", err) } - o.logger.Debug("env resolved", zap.String("ID", varsID)) + + resourceFetcher := GetResourceFetcher(o.logger, o.runnerRegistry) var resource any + if opts.DefinitionFile != "" { - f, err := fileutil.Read(opts.DefinitionFile) - if err != nil { - return ExitCodeGeneralError, fmt.Errorf("cannot read definition file %s: %w", opts.DefinitionFile, err) - } - df := f - o.logger.Debug("Definition file read", zap.String("absolutePath", df.AbsPath())) + resource, err = resourceFetcher.FetchWithDefinitionFile(ctx, opts.DefinitionFile) + } else { + resource, err = resourceFetcher.FetchWithID(ctx, opts.ID, opts.ResourceType) + } + if err != nil { + return ExitCodeGeneralError, err + } - df, err = o.injectLocalEnvVars(ctx, df) - if err != nil { - return ExitCodeGeneralError, fmt.Errorf("cannot inject local env vars: %w", err) - } + resourceType, err := resourcemanager.GetResourceType(resource) + if err != nil { + return ExitCodeGeneralError, fmt.Errorf("cannot extract type from resource: %w", err) + } - resource, err = r.Apply(ctx, df) - if err != nil { - return ExitCodeGeneralError, fmt.Errorf("cannot apply definition file: %w", err) - } - o.logger.Debug("Definition file applied", zap.String("updated", string(df.Contents()))) - } else { - o.logger.Debug("Definition file not provided, fetching resource by ID", zap.String("ID", opts.ID)) - resource, err = r.GetByID(ctx, opts.ID) - if err != nil { - return ExitCodeGeneralError, fmt.Errorf("cannot get resource by ID: %w", err) - } - o.logger.Debug("Resource fetched by ID", zap.String("ID", opts.ID), zap.Any("resource", resource)) + runner, err := o.runnerRegistry.Get(resourceType) + if err != nil { + return ExitCodeGeneralError, fmt.Errorf("cannot find runner for resource type %s: %w", resourceType, err) } var result RunResult - var ev varSets + var ev varset.VarSets // iterate until we have all env vars, // or the server returns an actual error for { runInfo := openapi.RunInformation{ VariableSetId: &varsID, - Variables: ev.toOpenapi(), - Metadata: getMetadata(), + Variables: ev.ToOpenapi(), + Metadata: metadata.GetMetadata(), RequiredGates: getRequiredGates(opts.RequiredGates), } - result, err = r.StartRun(ctx, resource, runInfo) + result, err = runner.StartRun(ctx, resource, runInfo) if err == nil { break } - if !errors.Is(err, missingVarsError{}) { + if !errors.Is(err, varset.MissingVarsError{}) { // actual error, return return ExitCodeGeneralError, fmt.Errorf("cannot run test: %w", err) } // missing vars error - ev = askForMissingVars([]varSet(err.(missingVarsError))) + ev = varset.AskForMissingVars([]varset.VarSet(err.(varset.MissingVarsError))) o.logger.Debug("filled variables", zap.Any("variables", ev)) } if opts.SkipResultWait { - fmt.Println(r.FormatResult(result, outputFormat)) + fmt.Println(runner.FormatResult(result, outputFormat)) return ExitCodeSuccess, nil } - result, err = o.waitForResult(ctx, r, result) + result, err = o.waitForResult(ctx, runner, result) if err != nil { return ExitCodeGeneralError, fmt.Errorf("cannot wait for test result: %w", err) } - fmt.Println(r.FormatResult(result, outputFormat)) + fmt.Println(runner.FormatResult(result, outputFormat)) - err = o.writeJUnitReport(ctx, r, result, opts.JUnitOuptutFile) + err = o.writeJUnitReport(ctx, runner, result, opts.JUnitOuptutFile) if err != nil { return ExitCodeGeneralError, fmt.Errorf("cannot write junit report: %w", err) } @@ -337,30 +342,6 @@ func (a orchestrator) writeJUnitReport(ctx context.Context, r Runner, result Run var source = "cli" -func getMetadata() map[string]string { - ci := cienvironment.DetectCIEnvironment() - if ci == nil { - return map[string]string{ - "source": source, - } - } - - metadata := map[string]string{ - "name": ci.Name, - "url": ci.URL, - "buildNumber": ci.BuildNumber, - "source": source, - } - - if ci.Git != nil { - metadata["branch"] = ci.Git.Branch - metadata["tag"] = ci.Git.Tag - metadata["revision"] = ci.Git.Revision - } - - return metadata -} - func getRequiredGates(gates []string) []openapi.SupportedGates { if len(gates) == 0 { return nil @@ -390,7 +371,7 @@ func HandleRunError(resp *http.Response, reqErr error) error { } if resp.StatusCode == http.StatusUnprocessableEntity { - return buildMissingVarsError(body) + return varset.BuildMissingVarsError(body) } if reqErr != nil { diff --git a/cli/runner/registry.go b/cli/runner/registry.go index af62ade05f..cdf0a3ace3 100644 --- a/cli/runner/registry.go +++ b/cli/runner/registry.go @@ -2,7 +2,9 @@ package runner import ( "fmt" + "strings" + "github.com/kubeshop/tracetest/cli/cmdutil" "go.uber.org/zap" ) @@ -16,7 +18,7 @@ func NewRegistry(logger *zap.Logger) Registry { return Registry{ runners: map[string]Runner{}, proxies: map[string]string{}, - logger: logger, + logger: cmdutil.GetLogger(), } } @@ -33,13 +35,13 @@ func (r Registry) RegisterProxy(proxyName, runnerName string) Registry { var ErrNotFound = fmt.Errorf("runner not found") func (r Registry) Get(name string) (Runner, error) { - runner, ok := r.runners[name] + runner, ok := r.runners[strings.ToLower(name)] if ok { return runner, nil // found runner, return it to the user } // fallback, check if the runner has a proxy - runnerName, ok := r.proxies[name] + runnerName, ok := r.proxies[strings.ToLower(name)] if !ok { return nil, ErrNotFound } diff --git a/cli/runner/report/reporter.go b/cli/runner/report/reporter.go new file mode 100644 index 0000000000..4e4e7e4604 --- /dev/null +++ b/cli/runner/report/reporter.go @@ -0,0 +1,71 @@ +package report + +import ( + "fmt" + + "github.com/pterm/pterm" +) + +type RunGroup struct { + ID string + Summary RunGroupSummary +} + +type RunGroupSummary struct { + Passed int + Failed int + InProgress int +} + +type Reporter struct { + runGroupID string + runGroupUrl string + + multi *pterm.MultiPrinter + groupSpinner *pterm.SpinnerPrinter + passedTestSpinner *pterm.SpinnerPrinter + failedTestSpinner *pterm.SpinnerPrinter + inProgressTestSpinner *pterm.SpinnerPrinter +} + +func NewReporter(runGroupID, runGroupURL string) *Reporter { + reporter := &Reporter{ + runGroupID: runGroupID, + runGroupUrl: runGroupURL, + multi: &pterm.DefaultMultiPrinter, + } + + reporter.groupSpinner, _ = pterm.DefaultSpinner.WithWriter(reporter.multi.NewWriter()).Start(fmt.Sprintf(`Parallel tests: %s in progress`, runGroupID)) + reporter.passedTestSpinner, _ = pterm.DefaultSpinner.WithWriter(reporter.multi.NewWriter()).WithStyle(pterm.FgGreen.ToStyle()).Start("0 tests passed") + reporter.failedTestSpinner, _ = pterm.DefaultSpinner.WithWriter(reporter.multi.NewWriter()).WithStyle(pterm.FgRed.ToStyle()).Start("0 tests failed") + reporter.inProgressTestSpinner, _ = pterm.DefaultSpinner.WithWriter(reporter.multi.NewWriter()).Start("0 tests in progress") + + return reporter +} + +func (r *Reporter) SetRunGroup(runGroup RunGroup) { + r.render(runGroup) +} + +func (r *Reporter) Start() { + r.multi.Start() +} + +func (r *Reporter) Stop() { + r.multi.Stop() +} + +func (r *Reporter) render(runGroup RunGroup) { + r.passedTestSpinner.UpdateText(fmt.Sprintf("%d tests passed", runGroup.Summary.Passed)) + r.failedTestSpinner.UpdateText(fmt.Sprintf("%d tests failed", runGroup.Summary.Failed)) + r.inProgressTestSpinner.UpdateText(fmt.Sprintf("%d tests in progress", runGroup.Summary.InProgress)) + + if runGroup.Summary.InProgress == 0 { + if runGroup.Summary.Failed > 0 { + r.groupSpinner.Fail(fmt.Sprintf(`Parallel tests: %s failed`, r.runGroupID)) + return + } + + r.groupSpinner.Success(fmt.Sprintf(`Parallel tests: %s succeed`, r.runGroupID)) + } +} diff --git a/cli/runner/resource_fetcher.go b/cli/runner/resource_fetcher.go new file mode 100644 index 0000000000..816b2ebed4 --- /dev/null +++ b/cli/runner/resource_fetcher.go @@ -0,0 +1,89 @@ +package runner + +import ( + "context" + "fmt" + + "github.com/kubeshop/tracetest/cli/pkg/fileutil" + "github.com/kubeshop/tracetest/cli/variable" + "go.uber.org/zap" +) + +type ResourceFetcher interface { + FetchWithDefinitionFile(context.Context, string) (any, error) + FetchWithID(context.Context, string, string) (any, error) +} + +type fetcher struct { + logger *zap.Logger + + runnerRegistry Registry +} + +func GetResourceFetcher(logger *zap.Logger, runnerRegistry Registry) ResourceFetcher { + return &fetcher{ + logger: logger, + runnerRegistry: runnerRegistry, + } +} + +var _ ResourceFetcher = &fetcher{} + +func (f *fetcher) FetchWithDefinitionFile(ctx context.Context, definitionFile string) (any, error) { + file, err := fileutil.Read(definitionFile) + if err != nil { + return nil, fmt.Errorf("cannot read definition file %s: %w", definitionFile, err) + } + df := file + f.logger.Debug("Definition file read", zap.String("absolutePath", df.AbsPath())) + + df, err = f.injectLocalEnvVars(df) + if err != nil { + return nil, fmt.Errorf("cannot inject local env vars: %w", err) + } + + runner, err := f.runnerRegistry.Get(file.Type()) + if err != nil { + return nil, fmt.Errorf("cannot get runner for type: %s: %w", file.Type(), err) + } + + resource, err := runner.Apply(ctx, df) + if err != nil { + return nil, fmt.Errorf("cannot apply definition file: %w", err) + } + f.logger.Debug("Definition file applied", zap.String("updated", string(df.Contents()))) + + return resource, nil +} + +func (f *fetcher) FetchWithID(ctx context.Context, resourceType string, resourceID string) (any, error) { + f.logger.Debug("Definition file not provided, fetching resource by ID", zap.String("ID", resourceID)) + + runner, err := f.runnerRegistry.Get(resourceType) + if err != nil { + return nil, fmt.Errorf("cannot get runner for resource type %s: %w", resourceType, err) + } + + resource, err := runner.GetByID(ctx, resourceID) + if err != nil { + return nil, fmt.Errorf("cannot get resource by ID: %w", err) + } + f.logger.Debug("Resource fetched by ID", zap.String("ID", resourceID), zap.Any("resource", resource)) + + return resource, nil +} + +func (f *fetcher) injectLocalEnvVars(df fileutil.File) (fileutil.File, error) { + variableInjector := variable.NewInjector(variable.WithVariableProvider( + variable.EnvironmentVariableProvider{}, + )) + + injected, err := variableInjector.ReplaceInString(string(df.Contents())) + if err != nil { + return df, fmt.Errorf("cannot inject local variable set: %w", err) + } + + df = fileutil.New(df.AbsPath(), []byte(injected)) + + return df, nil +} diff --git a/cli/runner/variableset_fetcher.go b/cli/runner/variableset_fetcher.go new file mode 100644 index 0000000000..ddd2195ae6 --- /dev/null +++ b/cli/runner/variableset_fetcher.go @@ -0,0 +1,75 @@ +package runner + +import ( + "context" + "errors" + "fmt" + + "github.com/kubeshop/tracetest/cli/openapi" + "github.com/kubeshop/tracetest/cli/pkg/fileutil" + "github.com/kubeshop/tracetest/cli/pkg/resourcemanager" + "go.uber.org/zap" + "gopkg.in/yaml.v2" +) + +type VariableSetFetcher interface { + Fetch(context.Context, string) (string, error) +} + +type internalVariableSetFetcher struct { + logger *zap.Logger + + variableSetClient resourcemanager.Client +} + +func GetVariableSetFetcher(logger *zap.Logger, variableSetClient resourcemanager.Client) VariableSetFetcher { + return &internalVariableSetFetcher{ + logger: logger, + variableSetClient: variableSetClient, + } +} + +var _ VariableSetFetcher = &internalVariableSetFetcher{} + +func (f *internalVariableSetFetcher) Fetch(ctx context.Context, varsID string) (string, error) { + if varsID == "" { + return "", nil // user have not defined variables, skipping it + } + + if !fileutil.IsFilePath(varsID) { + f.logger.Debug("varsID is not a file path", zap.String("vars", varsID)) + + // validate that env exists + _, err := f.variableSetClient.Get(ctx, varsID, resourcemanager.Formats.Get(resourcemanager.FormatYAML)) + if errors.Is(err, resourcemanager.ErrNotFound) { + return "", fmt.Errorf("variable set '%s' not found", varsID) + } + if err != nil { + return "", fmt.Errorf("cannot get variable set '%s': %w", varsID, err) + } + + f.logger.Debug("envID is valid") + + return varsID, nil + } + + file, err := fileutil.Read(varsID) + if err != nil { + return "", fmt.Errorf("cannot read environment set file %s: %w", varsID, err) + } + + f.logger.Debug("envID is a file path", zap.String("filePath", varsID), zap.Any("file", f)) + updatedEnv, err := f.variableSetClient.Apply(ctx, file, yamlFormat) + if err != nil { + return "", fmt.Errorf("could not read environment set file: %w", err) + } + + var vars openapi.VariableSetResource + err = yaml.Unmarshal([]byte(updatedEnv), &vars) + if err != nil { + f.logger.Error("error parsing json", zap.String("content", updatedEnv), zap.Error(err)) + return "", fmt.Errorf("could not unmarshal variable set json: %w", err) + } + + return vars.Spec.GetId(), nil +} diff --git a/cli/runner/var_set.go b/cli/varset/varset.go similarity index 62% rename from cli/runner/var_set.go rename to cli/varset/varset.go index ddd267df41..da95c92474 100644 --- a/cli/runner/var_set.go +++ b/cli/varset/varset.go @@ -1,17 +1,20 @@ -package runner +package varset import ( "fmt" "github.com/kubeshop/tracetest/cli/openapi" + "github.com/kubeshop/tracetest/cli/pkg/resourcemanager" "github.com/kubeshop/tracetest/cli/ui" ) -func askForMissingVars(missingVars []varSet) []varSet { +var jsonFormat = resourcemanager.Formats.Get(resourcemanager.FormatJSON) + +func AskForMissingVars(missingVars []VarSet) VarSets { ui.DefaultUI.Warning("Some variables are required by one or more tests") ui.DefaultUI.Info("Fill the values for each variable:") - filledVariables := make([]varSet, 0, len(missingVars)) + filledVariables := make([]VarSet, 0, len(missingVars)) for _, missingVar := range missingVars { answer := missingVar @@ -22,13 +25,13 @@ func askForMissingVars(missingVars []varSet) []varSet { return filledVariables } -type varSet struct { +type VarSet struct { Name string DefaultValue string UserValue string } -func (ev varSet) value() string { +func (ev VarSet) value() string { if ev.UserValue != "" { return ev.UserValue } @@ -36,9 +39,9 @@ func (ev varSet) value() string { return ev.DefaultValue } -type varSets []varSet +type VarSets []VarSet -func (evs varSets) toOpenapi() []openapi.VariableSetValue { +func (evs VarSets) ToOpenapi() []openapi.VariableSetValue { vars := make([]openapi.VariableSetValue, len(evs)) for i, ev := range evs { vars[i] = openapi.VariableSetValue{ @@ -50,9 +53,9 @@ func (evs varSets) toOpenapi() []openapi.VariableSetValue { return vars } -func (evs varSets) unique() varSets { +func (evs VarSets) Unique() VarSets { seen := make(map[string]bool) - vars := make(varSets, 0, len(evs)) + vars := make(VarSets, 0, len(evs)) for _, ev := range evs { if seen[ev.Name] { continue @@ -65,34 +68,34 @@ func (evs varSets) unique() varSets { return vars } -type missingVarsError varSets +type MissingVarsError VarSets -func (e missingVarsError) Error() string { - return fmt.Sprintf("missing variables: %v", []varSet(e)) +func (e MissingVarsError) Error() string { + return fmt.Sprintf("missing variables: %v", []VarSet(e)) } -func (e missingVarsError) Is(target error) bool { - _, ok := target.(missingVarsError) +func (e MissingVarsError) Is(target error) bool { + _, ok := target.(MissingVarsError) return ok } -func buildMissingVarsError(body []byte) error { +func BuildMissingVarsError(body []byte) error { var missingVarsErrResp openapi.MissingVariablesError err := jsonFormat.Unmarshal(body, &missingVarsErrResp) if err != nil { return fmt.Errorf("could not unmarshal response body: %w", err) } - missingVars := varSets{} + missingVars := VarSets{} for _, missingVarErr := range missingVarsErrResp.MissingVariables { for _, missingVar := range missingVarErr.Variables { - missingVars = append(missingVars, varSet{ + missingVars = append(missingVars, VarSet{ Name: missingVar.GetKey(), DefaultValue: missingVar.GetDefaultValue(), }) } } - return missingVarsError(missingVars.unique()) + return MissingVarsError(missingVars.Unique()) } diff --git a/testing/cli-e2etest/testscenarios/test/run_test_test.go b/testing/cli-e2etest/testscenarios/test/run_test_test.go index 16d46d7d6a..53974446f3 100644 --- a/testing/cli-e2etest/testscenarios/test/run_test_test.go +++ b/testing/cli-e2etest/testscenarios/test/run_test_test.go @@ -12,16 +12,13 @@ import ( ) func TestRunTestSuiteInsteadOfTest(t *testing.T) { - t.Run("should fail if test suite resource is selected", func(t *testing.T) { + t.Run("should fail as regular test even if test suite resource is selected", func(t *testing.T) { // setup isolated e2e environment env := environment.CreateAndStart(t) defer env.Close(t) cliConfig := env.GetCLIConfigPath(t) - // instantiate require with testing helper - require := require.New(t) - // Given I am a Tracetest CLI user // And I have my server recently created // And the datasource is already set @@ -32,8 +29,7 @@ func TestRunTestSuiteInsteadOfTest(t *testing.T) { command := fmt.Sprintf("run testsuite -f %s", testFil) result := tracetestcli.Exec(t, command, tracetestcli.WithCLIConfig(cliConfig)) - helpers.RequireExitCodeEqual(t, result, 1) - require.Contains(result.StdErr, "cannot apply Test to TestSuite resource") + helpers.RequireExitCodeEqual(t, result, 2) }) } diff --git a/testing/cli-e2etest/testscenarios/testsuite/run_testsuite_test.go b/testing/cli-e2etest/testscenarios/testsuite/run_testsuite_test.go index d7f999b154..60f3a0324d 100644 --- a/testing/cli-e2etest/testscenarios/testsuite/run_testsuite_test.go +++ b/testing/cli-e2etest/testscenarios/testsuite/run_testsuite_test.go @@ -11,16 +11,13 @@ import ( ) func TestRunTestSuite(t *testing.T) { - t.Run("should fail if test resource is selected", func(t *testing.T) { + t.Run("should fail as regular test suite even if test resource is selected", func(t *testing.T) { // setup isolated e2e environment env := environment.CreateAndStart(t) defer env.Close(t) cliConfig := env.GetCLIConfigPath(t) - // instantiate require with testing helper - require := require.New(t) - // Given I am a Tracetest CLI user // And I have my server recently created // And the datasource is already set @@ -31,8 +28,7 @@ func TestRunTestSuite(t *testing.T) { command := fmt.Sprintf("run test -f %s", testsuiteFile) result := tracetestcli.Exec(t, command, tracetestcli.WithCLIConfig(cliConfig)) - helpers.RequireExitCodeEqual(t, result, 1) - require.Contains(result.StdErr, "cannot apply TestSuite to Test resource") + helpers.RequireExitCodeEqual(t, result, 2) }) t.Run("should pass", func(t *testing.T) {