From 4f9435695ea4ce069ee5995743d2986acd1ca5e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adnan=20Rahi=C4=87?= Date: Wed, 7 Jun 2023 15:07:09 +0200 Subject: [PATCH] docs(cli+webui normalization): Match both CLI and Web UI features (#2646) * docs(cli+webui normalization): Added transactions and envs pages for webui * docs(cli+webui normalization): edit code blocks * docs(cli+webui normalization): added webui creating data stores, added cli running transactions * docs(cli+webui normalization): split cli/creating-tests into multiple pages * Update docs/docs/cli/creating-test-outputs.md Co-authored-by: Julianne Fermi * Update docs/docs/cli/creating-test-outputs.md Co-authored-by: Julianne Fermi * Update docs/docs/cli/creating-test-outputs.md Co-authored-by: Julianne Fermi * Update docs/docs/cli/running-tests.md Co-authored-by: Julianne Fermi * docs(cli+webui normalization): typo * docs(cli+webui normalization): added undef vars in CLI, create dedicated ad hoc testing page under concepts --------- Co-authored-by: Julianne Fermi --- docs/docs/cli/creating-data-stores.md | 2 + docs/docs/cli/creating-environments.md | 10 +- docs/docs/cli/creating-test-outputs.md | 79 +++++++++ docs/docs/cli/creating-test-specifications.md | 88 ++++++++++ docs/docs/cli/creating-tests.md | 166 ++---------------- docs/docs/cli/creating-transactions.md | 10 +- docs/docs/cli/running-tests.md | 49 ++++-- docs/docs/cli/running-transactions.md | 69 ++++++++ docs/docs/cli/undefined-variables.md | 56 ++++++ docs/docs/concepts/ad-hoc-testing.md | 49 ++++++ docs/docs/web-ui/creating-data-stores.md | 16 ++ docs/docs/web-ui/creating-environments.md | 23 +++ docs/docs/web-ui/creating-transactions.md | 31 ++++ docs/docs/web-ui/undefined-variables.md | 40 +---- docs/sidebars.js | 134 ++++++++++---- 15 files changed, 572 insertions(+), 250 deletions(-) create mode 100644 docs/docs/cli/creating-test-outputs.md create mode 100644 docs/docs/cli/creating-test-specifications.md create mode 100644 docs/docs/cli/running-transactions.md create mode 100644 docs/docs/cli/undefined-variables.md create mode 100644 docs/docs/concepts/ad-hoc-testing.md create mode 100644 docs/docs/web-ui/creating-data-stores.md diff --git a/docs/docs/cli/creating-data-stores.md b/docs/docs/cli/creating-data-stores.md index 5faae897f6..16bffad558 100644 --- a/docs/docs/cli/creating-data-stores.md +++ b/docs/docs/cli/creating-data-stores.md @@ -2,6 +2,8 @@ You might have multiple Tracetest instances that need to be connected to the same data stores. An easy way of sharing the configuration is by using a configuration file that can be applied to your Tracetest instance. +## Supported Trace Data Stores + ### Jaeger ```yaml diff --git a/docs/docs/cli/creating-environments.md b/docs/docs/cli/creating-environments.md index edf3f3e587..44a12d3582 100644 --- a/docs/docs/cli/creating-environments.md +++ b/docs/docs/cli/creating-environments.md @@ -1,4 +1,10 @@ -# Creating Environments +# Defining Environments as Text Files + +This page showcases how to create and edit environments with the CLI. + +:::tip +[To read more about environments check out environment concepts.](../concepts/environments.md) +::: Just like Data Stores, you can also manage your environments using the CLI and definition files. @@ -18,7 +24,7 @@ spec: In order to apply this configuration to your Tracetest instance, make sure to have your [CLI configured](./configuring-your-cli.md) and run: -``` +```sh tracetest apply environment -f ``` diff --git a/docs/docs/cli/creating-test-outputs.md b/docs/docs/cli/creating-test-outputs.md new file mode 100644 index 0000000000..42c16f39f0 --- /dev/null +++ b/docs/docs/cli/creating-test-outputs.md @@ -0,0 +1,79 @@ +# Defining Test Outputs in Text Files + +Outputs are really useful when running [Transactions](../concepts/transactions). They allow for exporting values from a test so they become available in the [Environment Variables](../concepts/environments.md) of the current transaction. + +## Outputs are Expression Results + +An output exports the result of an [Expression](../concepts/expressions) and assigns it to a name, so it can be injected into the environment variables of a running transaction. +A `selector` is needed only if the provided expression refers to a/some span/s attribute or meta attributes. + +It can be defined using the following YAML definition: + +```yaml +outputs: + - name: USER_ID + selector: span[name = "user creation"] + value: attr:myapp.users.created_id +``` + +The `value` attribute is an `expression` and is a very powerful tool. + +## Examples + +### Basic Expression + +You can output basic expressions: + +```yaml +outputs: + +- name: ARITHMETIC_RESULT + value: 1 + 1 + # results in ARITHMETIC_RESULT = 2 + +- name: INTERPOLATE_STRING + # assume PRE_EXISTING_VALUE=someValue from env vars + value: "the value ${env:PRE_EXISTING_VALUE} comes from the env var PRE_EXISTING_VALUE" + # results in INTERPOLATE_STRING = "the value someValue comes from the env var PRE_EXISTING_VALUE +``` + +### Extract a Value from a JSON + +Imagine a hypotetical `/users/create` endpoint that returns the full `user` object, including the new ID, when the operation is successful. + +```yaml +outputs: +- name: USER_ID + selector: span[name = "POST /user/create"] + value: attr:http.response.body | json_path '.id' +``` + +### Multiple Values + +Using the same hypotethical user creation endpoint, a user creation might result on multiple sql queries, for example: + +- `INSERT INTO users ...` +- `INSERT INTO permissions...` +- `SELECT remaining_users FROM accounts` +- `UPDATE accounts SET remaining_users ...` + +In this case, the service is instrumented so that each query generates a span of type `database`. +You can get a list of SQL operations: + +```yaml +outputs: +- name: SQL_OPS + selector: span[tracetest.span.type = "database"] + value: attr:sql.operation + # result: SQL_OPS = ["INSERT", "INSERT", "SELECT", "UPDATE"] +``` + +Since the value is an array, you can also apply filters to it: + +```yaml +outputs: +- name: LAST_SQL_OP + selector: span[tracetest.span.type = "database"] + value: attr:sql.operation | get_index 'last' + # result: LAST_SQL_OP = "INSERT" +``` diff --git a/docs/docs/cli/creating-test-specifications.md b/docs/docs/cli/creating-test-specifications.md new file mode 100644 index 0000000000..25c8269bd3 --- /dev/null +++ b/docs/docs/cli/creating-test-specifications.md @@ -0,0 +1,88 @@ +# Defining Test Specifications in Text Files + +Test Specifications may be added to a trace to set a value for a step in the trace to determine success or failure. + +## Assertions and Selectors + +Assertions are as important as how you trigger your test. Without them, your test is just a fancy way of executing a request using a CLI command. In this section, we will discuss how you can declare your assertions in your definition file. + +Before we start, there are two concepts that you must understand to write your tests: + +- [Selectors](../concepts/selectors.md) +- [Assertions](../concepts/assertions.md) + +### Selectors + +**Selectors** are queries that are executed against your trace tree and select a set of spans based on some attributes. They are responsible for defining which spans will be tested against your assertions. + +### Assertions + +**Assertions** are tests against a specific span based on its attributes. A practical example might be useful: + +Imagine you have to ensure that all your `database select statements` take `less than 500ms`. To write a test for that you must: + +1. Select all spans in your trace related to `select statements`. +2. Check if all those spans lasted `less than 500ms`. + +For the first task, we use a selector: `span[db.statement contains "SELECT"]`. While the second one is achieved by using an assertion: `attr:tracetest.span.duration < 500ms`. + +> **Note:** When asserting time fields, you can use the following time units: `ns` (nanoseconds), `us` (microseconds), `ms` (milliseconds), `s` (seconds), `m` (minutes), and `h` (hours). Instead of defining `attr:tracetest.span.duration <= 3600s`, you can set it as `attr:tracetest.span.duration <= 1h`. + +To write that in your test definition, you can define the following YAML definition: + +```yaml +specs: +- selector: span[db.statement contains "SELECT"] + assertions: + - attr:tracetest.span.duration < 500ms +``` + +As you probably noticed in the test definition structure, you can have multiple assertions for the same selector. This is useful to group related validations. For example, ensuring that all your HTTP calls are successful and take less than 1000ms: + +```yaml +specs: +- selector: span[tracetest.span.type="http"] + assertions: + - attr:http.status_code >= 200 + - attr:http.status_code < 300 + - attr:tracetest.span.duration < 1000ms +``` + +#### Referencing Other Fields from the Same Span + +You also can reference fields from the same span in your assertions. For example, you can define an assertion to ensure the output number is greater than the input number. + +```yaml +specs: +- selector: span[name = "my operation"] + assertions: + - attr:myapp.output > attr:myapp.input +``` + +You also can use basic arithmetic expressions in your assertions: + +```yaml +assertions: + - attr:myapp.output = myapp.input + 1 +``` + +:::note +This does not take into account the order of operators yet. So an expression `1 + 2 * 3` will be resolved as `9` instead of `7`. This will be fixed in future releases. +::: + +Available operations in an expression are: `+`, `-`, `*`, and `/`. + +For more information about selectors or assertions, take a look at the documentation for those topics. + +## Available Operations + +| Operator | Description | +| :------------- | ------------------------------------------------------------------------------------------------------------------------ | +| `=` | Check if two values are equal. | +| `!=` | Check if two values have different values. | +| `<` | Check if value from left side is smaller than the one on the right side of the operation. | +| `<=` | Check if value from left side is smaller or equal to the one on the right side of the operation. | +| `>` | Check if value from left side is larger than the one on the right side of the operation. | +| `>=` | Check if value from left side is larger or equal to the one on the right side of the operation. | +| `contains` | Check if value on the right side of the operation is contained inside of the value of the left side of the operation. | +| `not-contains` | Check if value on the right side of the operation is not contained inside of the value of the left side of the operation. | diff --git a/docs/docs/cli/creating-tests.md b/docs/docs/cli/creating-tests.md index 7934a7b2c2..5e57135e9e 100644 --- a/docs/docs/cli/creating-tests.md +++ b/docs/docs/cli/creating-tests.md @@ -9,9 +9,10 @@ As Tracetest is mainly a visual tool, this might make it difficult to update tes Imagine that you were assigned a ticket to improve your application database usage. You notice that every time a specific endpoint is called, your application executes `N+1` select statements on the database instead of only one statement. You probably already have a test in place to ensure the correct functionality of that endpoint: it inserts the necessary information into the database, calls that specific endpoint using our tool and ensures you get the expected results using the trace generated by your application. It works fine, but there is a problem. That test is managed by Tracetest on its server and the test cannot be changed until the new patch is deployed. Otherwise, if the test is run using a non-patched version of the application, the test would fail. To solve that, the best approach would be to enable developers to define their tests as text files and allow them to run those tests using a CLI, so you can integrate the execution of those tests to your existing CI pipeline. There are many benefits of this functionality for your tests: - - Peers can review your tests before merging them to the main branch. - - Ensure your test works before merging it to the main branch. - - Have different versions of the same test running in parallel in different branches, so you and your peers can work on the same code modules and update the same test without interfering with each other. + +- Peers can review your tests before merging them to the main branch. +- Ensure your test works before merging it to the main branch. +- Have different versions of the same test running in parallel in different branches, so you and your peers can work on the same code modules and update the same test without interfering with each other. ## Definition @@ -91,6 +92,7 @@ trigger: Currently, we support three authentication methods for HTTP requests: `basic authentication`, `api key`, and `bearer token`. Here is one example of each authentication method: **Basic Authentication** + ```yaml trigger: type: http @@ -105,6 +107,7 @@ trigger: ``` **API Key Authentication** + ```yaml trigger: type: http @@ -120,6 +123,7 @@ trigger: ``` **Bearer Token Authentication** + ```yaml trigger: type: http @@ -166,154 +170,10 @@ Available functions: | `creditCardExpDate()` | Generates a random credit card expiration date (mm/yy). | | `randomInt(min, max)` | Generates a random integer contained in the closed interval defined by [`min`, `max`]. | +:::tip +[Continue reading about Test Specs, here.](./creating-test-specifications.md) +::: -## Assertions - -Assertions are as important as how you trigger your test. Without them, your test is just a fancy way of executing a request using a CLI command. In this section, we will discuss how you can declare your assertions in your definition file. - -Before we start, there are two concepts that you must understand to write your tests: [selectors](../concepts/selectors) and assertions. - -**Selectors** are queries that are executed against your trace tree and select a set of spans based on some attributes. They are responsible for defining which spans will be tested against your assertions. - -**Assertions** are tests against a specific span based on its attributes. A practical example might be useful: - -Imagine you have to ensure that all your `database select statements` take `less than 500ms`. To write a test for that you must: - -1. Select all spans in your trace related to `select statements`. -2. Check if all those spans lasted `less than 500ms`. - -For the first task, we use a selector: `span[db.statement contains "SELECT"]`. While the second one is achieved by using an assertion: `attr:tracetest.span.duration < 500ms`. - -> **Note:** When asserting time fields, you can use the following time units: `ns` (nanoseconds), `us` (microseconds), `ms` (milliseconds), `s` (seconds), `m` (minutes), and `h` (hours). Instead of defining `attr:tracetest.span.duration <= 3600s`, you can set it as `attr:tracetest.span.duration <= 1h`. - -To write that in your test definition, you can define the following YAML definition: - -```yaml -specs: -- selector: span[db.statement contains "SELECT"] - assertions: - - attr:tracetest.span.duration < 500ms -``` - -As you probably noticed in the test definition structure, you can have multiple assertions for the same selector. This is useful to group related validations. For example, ensuring that all your HTTP calls are successful and take less than 1000ms: - -```yaml -specs: -- selector: span[tracetest.span.type="http"] - assertions: - - attr:http.status_code >= 200 - - attr:http.status_code < 300 - - attr:tracetest.span.duration < 1000ms -``` - -#### Referencing Other Fields from the Same Span** -You also can reference fields from the same span in your assertions. For example, you can define an assertion to ensure the output number is greater than the input number. - -```yaml -specs: -- selector: span[name = "my operation"] - assertions: - - attr:myapp.output > attr:myapp.input -``` - -You also can use basic arithmetic expressions in your assertions: -```yaml -assertions: - - attr:myapp.output = myapp.input + 1 -``` -> **Note:** This does not take into account the order of operators yet. So an expression `1 + 2 * 3` will be resolved as `9` instead of `7`. This will be fixed in future releases. - -Available operations in an expression are: `+`, `-`, `*`, and `/`. - -For more information about selectors or assertions, take a look at the documentation for those topics. - -### Available Operations - -| Operator | Description | -| :------------- | ------------------------------------------------------------------------------------------------------------------------ | -| `=` | Check if two values are equal. | -| `!=` | Check if two values have different values. | -| `<` | Check if value from left side is smaller than the one on the right side of the operation. | -| `<=` | Check if value from left side is smaller or equal to the one on the right side of the operation. | -| `>` | Check if value from left side is larger than the one on the right side of the operation. | -| `>=` | Check if value from left side is larger or equal to the one on the right side of the operation. | -| `contains` | Check if value on the right side of the operation is contained inside of the value of the left side of the operation. | -| `not-contains` | Check if value on the right side of the operation is not contained inside of the value of the left side of the operation. | - - -## Outputs - -Outputs are really useful when running [Transactions](../concepts/transactions). They allow for exporting values from a test so they become available in the [Environment Variables](../concepts/environments.md) of the current transaction. - -An ouptut exports the result of an [Expression](../concepts/expressions) and assigns it to a name, so it can be injected into the environment variables of a running transaction. -A `selector` is needed only if the provided expression refers to a/some span/s attribute or meta attributes. - -It can be defined using the following YAML definition: - -```yaml -outputs: - - name: USER_ID - selector: span[name = "user creation"] - value: attr:myapp.users.created_id -``` - -The `value` attribute is an `expression`, and is a very powerful tool. - -## Examples - -### Basic expression - -You can output basic expressions: - -```yaml -outputs: - -- name: ARITHMETIC_RESULT - value: 1 + 1 - # results in ARITHMETIC_RESULT = 2 - -- name: INTERPOLATE_STRING - # assume PRE_EXISTING_VALUE=someValue from env vars - value: "the value ${env:PRE_EXISTING_VALUE} comes from the env var PRE_EXISTING_VALUE" - # results in INTERPOLATE_STRING = "the value someValue comes from the env var PRE_EXISTING_VALUE -``` - -### Extract a Value from a JSON - -Imagine a hypotetical `/users/create` endpoint that returns the full `user` object, including the new ID, when the operation is successful. - -```yaml -outputs: -- name: USER_ID - selector: span[name = "POST /user/create"] - value: attr:http.response.body | json_path '.id' -``` - -### Multiple values - -Using the same hypotethical user creation endpoint, a user creation might result on multiple sql queries, for example: -- `INSERT INTO users ...` -- `INSERT INTO permissions...` -- `SELECT remaining_users FROM accounts` -- `UPDATE accounts SET remaining_users ...` - -In this case, the service is instrumented so that each query generates a span of type `database`. -You can get a list of SQL operations: - -```yaml -outputs: -- name: SQL_OPS - selector: span[tracetest.span.type = "database"] - value: attr:sql.operation - # result: SQL_OPS = ["INSERT", "INSERT", "SELECT", "UPDATE"] -``` - -Since the value is an array, you can also apply filters to it: - -```yaml -outputs: -- name: LAST_SQL_OP - selector: span[tracetest.span.type = "database"] - value: attr:sql.operation | get_index 'last' - # result: LAST_SQL_OP = "INSERT" -``` +:::tip +[Continue reading about Test Outputs, here.](./creating-test-outputs.md) +::: diff --git a/docs/docs/cli/creating-transactions.md b/docs/docs/cli/creating-transactions.md index 3292ed979d..a934f595fc 100644 --- a/docs/docs/cli/creating-transactions.md +++ b/docs/docs/cli/creating-transactions.md @@ -1,4 +1,10 @@ -# Creating Transactions +# Defining Transactions as Text Files + +This page showcases how to create and edit Transactions with the CLI. + +:::tip +[To read more about transactions check out transactions concepts.](../concepts/transactions.md) +::: Just like other structures of Tracetest, you can also manage your transactions using the CLI and definition files. @@ -18,7 +24,7 @@ spec: In order to apply this transaction to your Tracetest instance, make sure to have your [CLI configured](./configuring-your-cli.md) and run: -``` +```sh tracetest apply transaction -f ``` diff --git a/docs/docs/cli/running-tests.md b/docs/docs/cli/running-tests.md index 62580e3cbb..6ebd859127 100644 --- a/docs/docs/cli/running-tests.md +++ b/docs/docs/cli/running-tests.md @@ -1,23 +1,30 @@ # Running Tests From the Command Line Interface (CLI) -Once you have created a test, whether from the Tracetest UI or via a text editor, you will need the capabity to run it via the Command Line Interface (CLI) to integrate it into your CI/CD process or your local development workflow. The documentation for running a test via the CLI can be found here: [tracetest test run](./reference/tracetest_test_run.md). This page will provide some examples of using this command. + +Once you have created a test, whether from the Tracetest UI or via a text editor, you will need the capabity to run it via the Command Line Interface (CLI) to integrate it into your CI/CD process or your local development workflow. + +The documentation for running a test via the CLI can be found here: + +- [tracetest test run](./reference/tracetest_test_run.md): This page provides examples of using this command. ## Running Your First Test -To run a test, give the path to the test definition file with the '-d' option. This will launch a test, providing us with a link to the created test run. +To run a test, give the path to the test definition file with the `'-d'` option. This will launch a test and provide a link to the created test run. + +```sh +tracetest test run -d path/to/test.yaml ``` -tracetest test run -d path/to/test.yaml -w -``` -Output: -``` + +```text title="Output:" ✔ Pokeshop - Import (http://localhost:11633/test/4oI08rA4g/run/3/test) ``` -Now, let's run the same test but tell the CLI to wait for the test to complete running before returning. This will provide results from the test. -``` +Now, let's run the same test but tell the CLI to wait for the test to complete running before returning with the `'-w'` option. This will provide results from the test. + +```sh tracetest test run -d path/to/test.yaml -w ``` -Output: -``` + +```text title="Output:" ✘ Pokeshop - Import (http://localhost:11633/test/4oI08rA4g/run/12/test) ✔ Response should be ok ✔ #59ce1f4250482ba5 @@ -35,11 +42,12 @@ Output: ``` Running the same command with the '-o json' option would change the output from the default of human readable 'pretty' to 'json'. This can be useful when you wish to extract particular data from the response. This would look like: -``` + +```sh tracetest test run -d path/to/test.yaml -w - o json ``` -Output: -``` + +```json title="Output:" { "testRunWebUrl": "http://localhost:11633/test/4oI08rA4g/run/13/test", "results": { @@ -163,11 +171,14 @@ Output: ``` You can also opt to output the result as JUnit to a file. You would run the command with a -j option and a file name, ie: -``` + +```sh tracetest test run -d path/to/test.yaml -w -j junit.out ``` + The JUnit output file would then contain the JUnit result, for example: -``` + +```xml @@ -194,12 +205,14 @@ You can reference an existing environment using its id. For example, given this ![testenv](../img/show-environment-definition.png) We can run a test and specify that environment with this command: -``` + +```sh tracetest test run -d path/to/test.yaml -e testenv -w ``` You can also reference an environment resource file which will be used to create a new environment or update an existing one. For example, if you have a file named local.env with this content: -``` + +```yaml type: Environment spec: id: local.env @@ -211,7 +224,7 @@ spec: value: vileplume ``` -``` +```sh tracetest test run -d path/to/test.yaml -e path/to/local.env -w ``` diff --git a/docs/docs/cli/running-transactions.md b/docs/docs/cli/running-transactions.md new file mode 100644 index 0000000000..91abfb388d --- /dev/null +++ b/docs/docs/cli/running-transactions.md @@ -0,0 +1,69 @@ +# Running Transactions From the Command Line Interface (CLI) + +Once you have created a transaction, whether from the Tracetest UI or via a text editor, you will need the capabity to run it via the Command Line Interface (CLI) to integrate it into your CI/CD process or your local development workflow. + +The command to run a transaction is the same as running a test from the CLI. + +The documentation for running a test via the CLI can be found here: + +- [tracetest test run](./reference/tracetest_test_run.md): This page provides examples of using this command. + +## Running Your First Transaction + +To run a transaction, give the path to the transaction definition file with the `'-d'` option. This will launch a transaction, providing us with a link to the created transaction run. + +```sh +tracetest test run -d path/to/transaction.yaml +``` + +```text title="Output:" +✔ Pokemon Transaction (http://localhost:11633/transaction/xcGqfHl4g/run/3) +``` + +Now, let's run the same test but tell the CLI to wait for the test to complete running before returning with the `'-w'` option. This will provide results from the test. + +```sh +tracetest test run -d path/to/transaction.yaml -w +``` + +```text title="Output:" +✔ Pokemon Transaction (http://localhost:11633/transaction/xcGqfHl4g/run/3) + ✔ Pokeshop - Import (http://localhost:11633/test/XRHjfH_4R/run/4/test) + ✔ Pokeshop - List (http://localhost:11633/test/QvPjBH_4g/run/4/test) +``` + +## Running a Transaction That Uses Environment Variables + +There are two ways of referencing an environment when running a transaction. + +You can reference an existing environment using its id. For example, given this defined environment with an id of `'testenv'`: + +![testenv](../img/show-environment-definition.png) + +We can run a transaction and specify that environment with this command: + +```sh +tracetest test run -d path/to/transaction.yaml -e testenv -w +``` + +You can also reference an environment resource file which will be used to create a new environment or update an existing one. For example, if you have a file named `local.env` with this content: + +```yaml +type: Environment +spec: + id: local.env + name: local.env + values: + - key: POKEID + value: 45 + - key: POKENAME + value: vileplume +``` + +```sh +tracetest test run -d path/to/transaction.yaml -e path/to/local.env -w +``` + +If you use the environment resource approach, a new environment will be created in Tracetest. + +The second approach is very useful if you are running tests from a CI pipeline. diff --git a/docs/docs/cli/undefined-variables.md b/docs/docs/cli/undefined-variables.md new file mode 100644 index 0000000000..f14c760c89 --- /dev/null +++ b/docs/docs/cli/undefined-variables.md @@ -0,0 +1,56 @@ +# Undefined Variables + +When a user runs a test or a transaction, any variables that will be needed but are not defined will be prompted for: + +```sh +tracetest test run -d path/to/test.yaml +``` + +```text title="Output:" +WARNING Some variables are required by one or more tests +INFO Fill the values for each variable: +POKEID: +POKENAME: +``` + +Undefined variables are dependent on the environment selected and whether or not the variable is defined in the current environment. Select the environment to run the test or transaction by passing it into the test run command. + +```sh +tracetest list environment +``` + +```text title="Output:" + ID NAME DESCRIPTION +--------- --------- ------------- + testenv testenv testenv +``` + +```sh +tracetest get environment --id testenv +``` + +```text title="Output:" +--- +type: Environment +spec: + id: testenv + name: testenv + description: testenv + values: + - key: POKEID + value: "42" + - key: POKENAME + value: oddish +``` + +```sh +tracetest test run -d path/to/test.yaml -e testenv +``` + +```text title="Output:" +✔ Pokeshop - Import (http://your-domain:11633/test/XRHjfH_4R/run/XYZ/test) +``` + +:::tip +[Check out use-cases for using undefined variables here.](../concepts/ad-hoc-testing.md) +::: diff --git a/docs/docs/concepts/ad-hoc-testing.md b/docs/docs/concepts/ad-hoc-testing.md new file mode 100644 index 0000000000..4ad9ecf104 --- /dev/null +++ b/docs/docs/concepts/ad-hoc-testing.md @@ -0,0 +1,49 @@ +# Ad-hoc Testing + +This page showcases use-cases for undefined variables and how to enable ad-hoc testing by utilizing environments and undefined variabled. + +:::tip +[Check out how to configure ad-hoc testing with undefined variables with the **Web UI** here.](../web-ui/undefined-variables.md) +::: + +:::tip +[Check out how to configure ad-hoc testing with undefined variables with the **CLI** here.](../cli/undefined-variables.md) +::: + +## **Undefined Variables Use Cases** + +### **Supply Variable Value at Runtime** + +A user wants a test or transaction they can run on a particular user, order id, etc. that is configurable at run time. This makes running an adhoc test in an environment, even production, very easy and convenient. In this case, the user references the variable, but doesn't add it to the environment. Each time they run the test or transaction, they will be prompted for the unspecified variables. + +### **Supply Variable Value from a Previous Test** + +A user wants to define 3 tests as part of a transaction. The first test has an output variable and this output is used by the second test. They define the first test. They then define the second test and reference the variable value that is output from the first test. + +In Tracetest, undefined variables can be used in both the UI and CLI. + +## **Undefined Variables Transaction with Multiple Tests Example** + +1. Create an HTTP Pokemon list test that uses environment variables for hostname and the SKIP query parameter: + +![Create Pokemon List](../img/pokeshop-list.png) + +2. Within the test, create test spec assertions that use environment variables for comparators, something like: `http.status_code = "${env:STATUS_CODE}"`: + +![Create Test Spec Assertionsl](../img/create-test-spec-assertions.png) + +3. Create a GRPC Pokemon add test that uses environment variables for hostname and Pokemon name: + +![Create GRPC](../img/create-grpc.png) + +4. Create an output from this test for the SKIP variable that could come from anywhere in the trace: + +![Test Output](../img/test-output.png) + +5. Now, you can create a transaction with the two tests - first, add the list test, then the add test, and then the list test again: + +![Create Transaction](../img/create-transaction.png) + +6. From here you can input the values for the undefined variables and complete your trace: + +![Input Values](../img/input-values.png) \ No newline at end of file diff --git a/docs/docs/web-ui/creating-data-stores.md b/docs/docs/web-ui/creating-data-stores.md new file mode 100644 index 0000000000..c8feef5d3d --- /dev/null +++ b/docs/docs/web-ui/creating-data-stores.md @@ -0,0 +1,16 @@ +# Configuring Data Stores + +This page shows how to configure data stores in the Tracetest Web UI. + +Open the **settings** and select the **Configure Data Store** tab. + +![](https://res.cloudinary.com/djwdcmwdz/image/upload/v1685971832/docs/Jaeger-settings-1477bdfe17b0675850681e0cfb85859a_ziy7un.png) + +To help you configure the connection, you can use the **Test Connection** button to validate if the connection is successful and Tracetest can fetch traces from the data store. + +![](https://res.cloudinary.com/djwdcmwdz/image/upload/v1685972024/docs/demo.tracetest.io_transaction_QZ3ejgl4R_run_2_3_wrwikl.png) + +## Supported Trace Data Stores + +Select from the [list of available data stores](../configuration/overview.md) and configure the connection. + diff --git a/docs/docs/web-ui/creating-environments.md b/docs/docs/web-ui/creating-environments.md index e69de29bb2..1448d19297 100644 --- a/docs/docs/web-ui/creating-environments.md +++ b/docs/docs/web-ui/creating-environments.md @@ -0,0 +1,23 @@ +# Creating Environments + +This page showcases how to create and edit environments in the Web UI. + +:::tip +[To read more about environments check out environment concepts.](../concepts/environments.md) +::: + +![](https://res.cloudinary.com/djwdcmwdz/image/upload/v1685967319/docs/demo.tracetest.io_transaction_QZ3ejgl4R_run_2_oproo4.png) + +On the environments page, click **Create Environment**. + +![](https://res.cloudinary.com/djwdcmwdz/image/upload/v1685967361/docs/demo.tracetest.io_transaction_QZ3ejgl4R_run_2_1_omab6p.png) + +To create an environment, click **Create**. + +You can also edit an environment by clicking on the three dots and clicking **Edit**. + +![](https://res.cloudinary.com/djwdcmwdz/image/upload/v1685967449/docs/Screenshot_2023-06-05_at_14.17.25_rmirjt.png) + +To export a YAML definition of an environment, click **Environment Definition**: + +![](https://res.cloudinary.com/djwdcmwdz/image/upload/v1685967413/docs/demo.tracetest.io_transaction_QZ3ejgl4R_run_2_2_l4tajt.png) diff --git a/docs/docs/web-ui/creating-transactions.md b/docs/docs/web-ui/creating-transactions.md index e69de29bb2..0ff4b7b0bc 100644 --- a/docs/docs/web-ui/creating-transactions.md +++ b/docs/docs/web-ui/creating-transactions.md @@ -0,0 +1,31 @@ +# Creating Transactions + +This page showcases how to create and edit Transactions in the Web UI. + +:::tip +[To read more about transactions check out transactions concepts.](../concepts/transactions.md) +::: + +![Main Screen](../img/main-screen-0.11.png) + +Click the **Create** button and select **Create New Transaction** in the drop down: + +![Create a Test Button](../img/create-button-0.11.png) + +Give your transaction a name, and click **Next**: + +![Name the Transaction](https://res.cloudinary.com/djwdcmwdz/image/upload/v1685712802/docs/beta.tracetest.io__page_1_jynf6o.png) + +Next, select which tests to run in the transaction and click **Create & Run**: + +![Select Tests in Transaction](https://res.cloudinary.com/djwdcmwdz/image/upload/v1685712954/docs/beta.tracetest.io__page_1_1_agjvg0.png) + +The transaction will start: + +![Running Transaction](https://res.cloudinary.com/djwdcmwdz/image/upload/v1685713352/docs/beta.tracetest.io__page_1_2_oqwazx.png) + +When the transaction is finished, you will get the following results: + +![Finished Transaction](https://res.cloudinary.com/djwdcmwdz/image/upload/v1685713712/docs/demo.tracetest.io__x0o1gu.png) + +You can now view individual [Test Results](test-results.md) executed by the transaction by clicking on any of the tests in the list. diff --git a/docs/docs/web-ui/undefined-variables.md b/docs/docs/web-ui/undefined-variables.md index b5648863f5..f0c768eff0 100644 --- a/docs/docs/web-ui/undefined-variables.md +++ b/docs/docs/web-ui/undefined-variables.md @@ -8,40 +8,6 @@ Undefined variables are dependent on the environment selected and whether or not ![Select Environment Drop Down](../img/select-environment-drop-down.png) -## **Undefined Variables Use Cases** - -### **Supply Variable Value at Runtime** - -A user wants a test or transaction they can run on a particular user, order id, etc. that is configurable at run time. This makes running an adhoc test in an environment, even production, very easy and convenient. In this case, the user references the variable, but doesn't add it to the environment. Each time they run the test or transaction, they will be prompted for the unspecified variables. - -### **Supply Variable Value from a Previous Test** - -A user wants to define 3 tests as part of a transaction. The first test has an output variable and this output is used by the second test. They define the first test. They then define the second test and reference the variable value that is output from the first test. - -In Tracetest, undefined variables can be used in both the UI and CLI. - -## **Undefined Variables Transaction with Multiple Tests Example** - -1. Create an HTTP Pokemon list test that uses environment variables for hostname and the SKIP query parameter: - -![Create Pokemon List](../img/pokeshop-list.png) - -2. Within the test, create test spec assertions that use environment variables for comparators, something like: http.status_code = "${env:STATUS_CODE}": - -![Create Test Spec Assertionsl](../img/create-test-spec-assertions.png) - -3. Create a GRPC Pokemon add test that uses environment variables for hostname and Pokemon name: - -![Create GRPC](../img/create-grpc.png) - -4. Create an output from this test for the SKIP variable that could come from anywhere in the trace: - -![Test Output](../img/test-output.png) - -5. Now, you can create a transaction with the two tests - first, add the list test, then the add test, and then the list test again: - -![Create Transaction](../img/create-transaction.png) - -6. From here you can input the values for the undefined variables and complete your trace: - -![Input Values](../img/input-values.png) \ No newline at end of file +:::tip +[Check out use-cases for using undefined variables here.](../concepts/ad-hoc-testing.md) +::: diff --git a/docs/sidebars.js b/docs/sidebars.js index 07d32ebb4b..2e62276ef3 100644 --- a/docs/sidebars.js +++ b/docs/sidebars.js @@ -238,6 +238,11 @@ const sidebars = { id: "concepts/transactions", label: "Transactions", }, + { + type: "doc", + id: "concepts/ad-hoc-testing", + label: "Ad-hoc Testing", + }, { type: "doc", id: "concepts/versioning", @@ -251,33 +256,54 @@ const sidebars = { items: [ { type: "doc", - id: "web-ui/creating-tests", - label: "Creating Tests", - }, - { - type: "doc", - id: "web-ui/creating-test-specifications", - label: "Creating Test Specifications", - }, - { - type: "doc", - id: "web-ui/creating-test-outputs", - label: "Creating Test Outputs", + id: "web-ui/creating-data-stores", + label: "Configuring Data Stores", }, { - type: "doc", - id: "web-ui/test-results", - label: "Test Results", + type: "category", + label: "Configuring Tests", + items: [ + { + type: "doc", + id: "web-ui/creating-tests", + label: "Creating Tests", + }, + { + type: "doc", + id: "web-ui/creating-test-specifications", + label: "Creating Test Specifications", + }, + { + type: "doc", + id: "web-ui/creating-test-outputs", + label: "Creating Test Outputs", + }, + { + type: "doc", + id: "web-ui/test-results", + label: "Test Results", + }, + { + type: "doc", + id: "web-ui/exporting-tests", + label: "Exporting Tests", + }, + { + type: "doc", + id: "web-ui/undefined-variables", + label: "Using Undefined Variables", + }, + ], }, { type: "doc", - id: "web-ui/exporting-tests", - label: "Exporting Tests", + id: "web-ui/creating-transactions", + label: "Creating Transactions", }, { type: "doc", - id: "web-ui/undefined-variables", - label: "Using Undefined Variables", + id: "web-ui/creating-environments", + label: "Creating Environments", }, ], }, @@ -293,33 +319,65 @@ const sidebars = { { type: "doc", id: "cli/creating-data-stores", - label: "Creating Data Stores", - }, - { - type: "doc", - id: "cli/creating-environments", - label: "Creating Environments", + label: "Configuring Data Stores", }, { - type: "doc", - id: "cli/creating-tests", - label: "Creating Tests", + type: "category", + label: "Configuring Tests", + items: [ + { + type: "doc", + id: "cli/creating-tests", + label: "Creating Tests", + }, + { + type: "doc", + id: "cli/creating-test-specifications", + label: "Creating Test Specifications", + }, + { + type: "doc", + id: "cli/creating-test-outputs", + label: "Creating Test Outputs", + }, + { + type: "doc", + id: "cli/running-tests", + label: "Running Tests", + }, + { + type: "doc", + id: "cli/undefined-variables", + label: "Using Undefined Variables", + }, + // { + // type: "doc", + // id: "cli/exporting-tests", + // label: "Exporting tests", + // }, + ], }, { - type: "doc", - id: "cli/running-tests", - label: "Running Tests", + type: "category", + label: "Configuring Transactions", + items: [ + { + type: "doc", + id: "cli/creating-transactions", + label: "Creating Transactions", + }, + { + type: "doc", + id: "cli/running-transactions", + label: "Running Transactions", + }, + ] }, { type: "doc", - id: "cli/creating-transactions", - label: "Creating Transactions", + id: "cli/creating-environments", + label: "Creating Environments", }, - // { - // type: "doc", - // id: "cli/exporting-tests", - // label: "Exporting tests", - // }, { type: "category", label: "Reference",