diff --git a/README.md b/README.md index aed5575..c15674c 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,11 @@ - Edit theme design & layout in `themes/middleware` - Edit theme configuration in `config.toml` -Local : `hugo server -D` +Open local server : +- `hugo server -D` + +Build static files: +- `hugo` ## Maintainers diff --git a/config.toml b/config.toml index fdb6cb6..504dd71 100644 --- a/config.toml +++ b/config.toml @@ -68,6 +68,18 @@ hasChildren = true url = "core-concept/workflow" weight = 3 + [[menu.main]] + parent = "Core concepts" + name = "HTTP Hook" + url = "core-concept/http-hook" + weight = 4 + + [[menu.main]] + parent = "Core concepts" + name = "HTTP API" + url = "core-concept/http-api" + weight = 5 + [[menu.main]] weight = 3 name = "Features" diff --git a/content/core-concept/http-api/_index.en.md b/content/core-concept/http-api/_index.en.md new file mode 100644 index 0000000..8cc4002 --- /dev/null +++ b/content/core-concept/http-api/_index.en.md @@ -0,0 +1,150 @@ +--- +title: "HTTP API" +date: 2020-07-12T15:21:02+02:00 +draft: true +type: "component" +logo: "pipeline" +description: "Data stream processing at high rate and low memory consuming" +weight: 5 +--- + +- [What is it ?](#what-is-it-) +- [Installation](#installation) +- [Basic usage](#basic-usage) +- [Advanced usage](#advanced-usage) + - [Adding JWT Authorization](#adding-json-web-token-jwt-authorization) + - [Adding Basic HTTP Authorization](#adding-basic-http-authorization) + +--- + +## What is it ? + +This package allows you to create an API that will serve multiple endpoints. + +The goal is to be able to send data to these endpoints, to process it in a series of steps. + +## Installation + +``` +composer require php-etl/workflow +``` + +## Basic usage + +To define your HTTP API, you need to specify a root `path`, and one or multiple `routes` under that root: + +```yaml +version: '0.3' +satellites: + my_satellite: + label: 'Example of an api' + filesystem: + path: build + composer: + require: + - "middlewares/uuid:dev-master" + - "middlewares/base-path:dev-master" + - "middlewares/request-handler:dev-master" + - "middlewares/fast-route:dev-master" + - "laminas/laminas-diactoros" + - "laminas/laminas-httphandlerrunner" + - "nyholm/psr7-server" + - "nyholm/psr7" + - "php-etl/pipeline" + - "php-etl/satellite" + - "php-etl/api-runtime" + - "php-etl/mapping-contracts" + http_api: + name: 'My HTTP API' # Optional + path: /my-api + routes: + - route: /transform + name: 'A route to transform my products' # Optional + method: 'post' # Optional. Default: "post" + # Possible values: "get", "post", "put", "delete", "patch", "head" + expression: 'input' + pipeline: + steps: + - fastmap: + map: + - field: '[sku]' + copy: '[product_name]' + - field: '[id]' + copy: '[product_code]' + - csv: + loader: + file_path: 'output.csv' +``` + +After [building](../../getting-started/compilation) the satellite, start a server in the path `build/`: + +```bash +bin/satellite run:api build/ +``` + +You can then send POST requests containing the data be processed to `http://localhost:8000/my-api/transform` + +```yaml +# input: +[ + { product_name: 'test_1', product_code: 861 }, + { product_name: 'test_2', product_code: 862 } +] + +# response: +{"message":{"accept":4,"reject":0,"error":0},"server":"my-computer.local"} + +# output.csv: +sku;id +test_1;861 +test_2;862 +``` + +## Advanced usage + +### Adding JSON Web Token (JWT) Authorization + +```yaml +# ... + composer: + require: + - "tuupola/slim-jwt-auth" +# ... + http_api: + authorization: + jwt: + secret: 'mysecret' +``` + +With this config, each requests will need the header __Authorization__: +|header|value| +|-|-| +|`Authorization`|`Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6Ik`[...]| + +The string after "Bearer" is the token, generated from the secret phrase. This site can be used to generate a token from your own secret: [https://jwt.io](https://jwt.io) + +### Adding Basic HTTP Authorization + +```yaml +# ... + composer: + require: + - "tuupola/slim-basic-auth" +# ... + http_api: + authorization: + basic: + - user: john + password: mypassword + - user: bill + password: otherpassword +``` + +The `basic` node is an array, and can contain multiple user/password pairs. + +With this configuration, each requests will need the header __Authorization__: +|header|value| +|-|-| +|`Authorization`|`Basic am9objpteXBhc3N3b3Jk`| + +The string after "Basic" is the combination `user:password` encoded in Base64. diff --git a/content/core-concept/http-hook/_index.en.md b/content/core-concept/http-hook/_index.en.md new file mode 100644 index 0000000..d2150a6 --- /dev/null +++ b/content/core-concept/http-hook/_index.en.md @@ -0,0 +1,145 @@ +--- +title: "HTTP Hook" +date: 2020-07-12T15:21:02+02:00 +draft: true +type: "component" +logo: "pipeline" +description: "Data stream processing at high rate and low memory consuming" +weight: 4 +--- + +- [What is it ?](#what-is-it-) +- [Installation](#installation) +- [Basic usage](#basic-usage) +- [Advanced usage](#advanced-usage) + - [Adding JWT Authorization](#adding-json-web-token-jwt-authorization) + - [Adding Basic HTTP Authorization](#adding-basic-http-authorization) + +--- + +## What is it ? + +This package allows you to create an API that will serve a single endpoint. + +The goal is to be able to send data to this endpoint, to process it in a series of steps. + +## Installation + +``` +composer require php-etl/workflow +``` + +## Basic usage + +Your HTTP Hook will serve the route set in the option `path`: + +```yaml +version: '0.3' +satellites: + my_satellite: + label: 'Example of a hook' + filesystem: + path: build + composer: + require: + - "middlewares/uuid:dev-master" + - "middlewares/base-path:dev-master" + - "middlewares/request-handler:dev-master" + - "middlewares/fast-route:dev-master" + - "laminas/laminas-diactoros" + - "laminas/laminas-httphandlerrunner" + - "nyholm/psr7-server" + - "nyholm/psr7" + - "php-etl/pipeline" + - "php-etl/satellite" + - "php-etl/api-runtime" + - "php-etl/mapping-contracts" + http_hook: + name: 'My HTTP Hook' # Optional + path: /my-hook + expression: 'input' + pipeline: + steps: + - fastmap: + map: + - field: '[sku]' + copy: '[product_name]' + - field: '[id]' + copy: '[product_code]' + - csv: + loader: + file_path: 'output.csv' +``` + +After [building](../../getting-started/compilation) the satellite, start a server in the path `build/`: + +```bash +bin/satellite run:hook build/ +``` + +You can then send POST requests containing the data be processed to `http://localhost:8000/my-hook` + +```yaml +# input: +[ + { product_name: 'test_1', product_code: 861 }, + { product_name: 'test_2', product_code: 862 } +] + +# response: +{"message":{"accept":4,"reject":0,"error":0},"server":"my-computer.local"} + +# output.csv: +sku;id +test_1;861 +test_2;862 +``` + +## Advanced usage + +### Adding JSON Web Token (JWT) Authorization + +```yaml +# ... + composer: + require: + - "tuupola/slim-jwt-auth" +# ... + http_hook: + authorization: + jwt: + secret: 'mysecret' +``` + +With this config, each requests will need the header __Authorization__: +|header|value| +|-|-| +|`Authorization`|`Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6Ik`[...]| + +The string after "Bearer" is the token, generated from the secret phrase. This site can be used to generate a token from your own secret: [https://jwt.io](https://jwt.io) + +### Adding Basic HTTP Authorization + +```yaml +# ... + composer: + require: + - "tuupola/slim-basic-auth" +# ... + http_hook: + authorization: + basic: + - user: john + password: mypassword + - user: bill + password: otherpassword +``` + +The `basic` node is an array, and can contain multiple user/password pairs. + +With this configuration, each requests will need the header __Authorization__: +|header|value| +|-|-| +|`Authorization`|`Basic am9objpteXBhc3N3b3Jk`| + +The string after "Basic" is the combination `user:password` encoded in Base64. diff --git a/content/core-concept/satellite/_index.en.md b/content/core-concept/satellite/_index.en.md index d2dbd19..eb53b31 100644 --- a/content/core-concept/satellite/_index.en.md +++ b/content/core-concept/satellite/_index.en.md @@ -1,7 +1,6 @@ --- title: "Creating Satellite" date: 2020-07-12T15:21:02+02:00 -weight: 1 draft: false type: "component" logo: "lambda" @@ -17,8 +16,6 @@ weight: 1 - [Using system](#using-system) - [Configure composer](#configure-composer) - [Setting up the runtime](#setting-up-the-runtime) - - [Using Pipeline](#using-pipeline) - - [Using Workflow](#using-workflow) - [Configuration formats](#configuration-formats) - Building - [Building locally](#build-your-satellite-locally) @@ -57,13 +54,13 @@ satellites: ### Setting up the Adapter -Your next step, you should declare the docker image, or the file, on which you want to build the satellite. +Next, you should declare the Docker image or the directory inside which you want to build the satellite. If you are using [Gyroscops Cloud](https://gyroscops.com), you can ignore this step and directly go to [the next chapter](#configure-composer). #### Using Docker -To use a docker image to build your satellite, implement the `docker` key with its configuration options : +To use a Docker image to build your satellite, implement the `docker` key with its configuration options : - `from` : determines the image on which your code will run - `workdir` : define the working directory of a Docker container @@ -88,7 +85,7 @@ satellites: {{< /tabs >}} Here, we chose to use the `php:8.0-cli-alpine` base image on which our code will be executed. -You could use any docker image of your choice, however you will need to have a PHP runtime +You could use any Docker image of your choice, however you will need to have a PHP runtime available, in a compatible version: >=8.0 with the CLI SAPI. #### Using the file system @@ -225,20 +222,17 @@ satellites: Now that we have made our environment prepared for our satellite, we will declare the way we want our pipeline to handle our data flows. -There are 4 types of runtimes, depending on your needs you will have to choose one of: - * `http-api`: the satellite will be operating an API, on which several URL routes can be registered. `http-api` is used for REST API. - * `http-hook`: the satellite will be operating an API on a single URL route. `http-hook` is used for webhooks. A webhook is a POST request sent to a URL. It's considered to be -a means for one application to provide other applications with real-time information - * `pipeline`: the satellite will be operating a data pipeline, executed in the backend that can be executed as a cron job. - * `workflow`: the satellite will be orchestrating more than one data pipeline, executed in the backend that can be executed as a cron job - -#### Using Pipeline - -Please visit the [Pipeline documentation page](../pipeline) to find out how to set up your pipeline. +There are 4 types of runtimes, you will have to choose one depending on your needs: +| name | description | details | +|-|-|-| +| pipeline | The satellite will be operating a data pipeline, executed in the backend that can be executed as a cron job. | [Pipeline documentation page](../pipeline) | +| workflow | The satellite will be orchestrating multiple pipelines, executed in the backend that can be executed as a cron job | [Workflow documentation page](../workflow) | +| http_hook | The satellite will be operating an API on a single URL route. `http_hook` is used for webhooks. A webhook is a POST request sent to a URL. It's considered to be a mean for one application to provide other applications with real-time information | [HTTP Hook documentation page](../http-hook) | +| http_api | The satellite will be operating an API on multiple URL routes. `http_api` is used for REST API. | [HTTP API documentation page](../http-api) | -{{< tabs name="dataflows" >}} +{{< tabs name="runtimes" >}} -{{< tab name="YAML" codelang="yaml" >}} +{{< tab name="pipeline" codelang="yaml" >}} version: '0.3' satellites: my_satellite: @@ -246,49 +240,25 @@ satellites: # ... pipeline: steps: - - akeneo: - enterprise: true - extractor: - type: productModel - method: all - search: - - { field: enabled, operator: '=', value: true } - - { field: completeness, operator: '>', value: 70, scope: ecommerce } - - { field: completeness, operator: '<', value: 85, scope: ecommerce } - - { field: categories, operator: IN, value: winter_collection } - - { field: family, operator: IN, value: [camcorders, digital_cameras] } - logger: - type: 'stderr' - - fastmap: - map: - - field: '[sku]' - copy: '[sku]' - - field: '[title]' - expression: 'input["sku"] ~" | "~ input["name"]' - - field: '[name]' - copy: '[name]' - - field: '[staticValue]' - constant: 'Lorem ipsum dolor sit amet, consectetur adipiscing elit. Curabitur mollis efficitur justo, id facilisis elit venenatis et. Sed fermentum posuere convallis. Phasellus lectus neque, bibendum sit amet enim imperdiet, dignissim blandit nisi. Donec nec neque nisi. Vivamus luctus facilisis nibh id rhoncus. Vestibulum eget facilisis tortor. Etiam at cursus enim, vitae mollis ex. Proin at velit at erat bibendum ultricies. Duis ut velit malesuada, placerat nisl a, ultrices tortor.' - - field: '[foo]' - expression: 'input' - list: - - field: '[bar]' - copy: '[bar]' - - csv: - loader: - file_path: output.csv - delimiter: ',' - enclosure: '"' - escape: '\' + - akeneo: + extractor: + type: category + method: all + client: + api_url: '@=env("AKENEO_URL")' + client_id: '@=env("AKENEO_CLIENT_ID")' + secret: '@=env("AKENEO_CLIENT_SECRET")' + username: '@=env("AKENEO_USERNAME")' + password: '@=env("AKENEO_PASSWORD")' + - csv: + loader: + file_path: categories.csv + delimiter: ',' + enclosure: '"' + escape: '\\' {{< /tab >}} -{{< /tabs >}} - -#### Using Workflow - -Please visit the [Workflow documentation page](../workflow) to find out how to set up your workflow. - -```yaml +{{< tab name="workflow" codelang="yaml" >}} version: '0.3' satellites: my_satellite: @@ -296,8 +266,7 @@ satellites: # ... workflow: jobs: - - name: 'Lorem ipsum dolor' - pipeline: + - pipeline: steps: - akeneo: extractor: @@ -315,27 +284,86 @@ satellites: delimiter: ',' enclosure: '"' escape: '\\' - my_second_satellite: - label: 'My second Satellite' - pipeline: - steps: - - akeneo: - extractor: - type: product - method: all - client: - api_url: '@=env("AKENEO_URL")' - client_id: '@=env("AKENEO_CLIENT_ID")' - secret: '@=env("AKENEO_CLIENT_SECRET")' - username: '@=env("AKENEO_USERNAME")' - password: '@=env("AKENEO_PASSWORD")' - - csv: - loader: - file_path: products.csv - delimiter: ',' - enclosure: '"' - escape: '\' -``` + - pipeline: + steps: + - akeneo: + extractor: + type: product + method: all + client: + api_url: '@=env("AKENEO_URL")' + client_id: '@=env("AKENEO_CLIENT_ID")' + secret: '@=env("AKENEO_CLIENT_SECRET")' + username: '@=env("AKENEO_USERNAME")' + password: '@=env("AKENEO_PASSWORD")' + - csv: + loader: + file_path: products.csv + delimiter: ',' + enclosure: '"' + escape: '\' +{{< /tab >}} + +{{< tab name="http_hook" codelang="yaml" >}} +version: '0.3' +satellites: + my_satellite: + label: 'My first Satellite' + # ... + http_hook: + path: /my-hook + expression: 'input' + pipeline: + steps: + - fastmap: + map: + - field: '[sku]' + copy: '[product_name]' + - field: '[id]' + copy: '[product_code]' + - csv: + loader: + file_path: 'output.csv' +{{< /tab >}} + +{{< tab name="http_api" codelang="yaml" >}} +version: '0.3' +satellites: + my_satellite: + label: 'My first Satellite' + # ... + http_api: + path: /my-api + routes: + - route: /category + expression: 'input' + pipeline: + steps: + - fastmap: + map: + - field: '[code]' + copy: '[category_name]' + - field: '[id]' + copy: '[category_code]' + - csv: + loader: + file_path: 'category.csv' + - route: /product + expression: 'input' + pipeline: + steps: + - fastmap: + map: + - field: '[sku]' + copy: '[product_name]' + - field: '[id]' + copy: '[product_code]' + - csv: + loader: + file_path: 'product.csv' +{{< /tab >}} + +{{< /tabs >}} ### Configuration formats diff --git a/themes/middleware/assets/css/style.css b/themes/middleware/assets/css/style.css index 06f43e3..692f635 100644 --- a/themes/middleware/assets/css/style.css +++ b/themes/middleware/assets/css/style.css @@ -637,8 +637,7 @@ h1.single-title { .content h1, .content h2, -.content h3, -.content h4 { +.content h3 { padding: 10px 0; position: sticky; top: 95px;