From b773f2df64f1293f2cdeb25f8f5af6f6e1012229 Mon Sep 17 00:00:00 2001 From: he2ss Date: Fri, 25 Oct 2024 11:50:03 +0200 Subject: [PATCH 1/4] add http datasource --- crowdsec-docs/docs/data_sources/http.md | 170 ++++++++++++++++++++++++ 1 file changed, 170 insertions(+) create mode 100644 crowdsec-docs/docs/data_sources/http.md diff --git a/crowdsec-docs/docs/data_sources/http.md b/crowdsec-docs/docs/data_sources/http.md new file mode 100644 index 000000000..1cbe08fde --- /dev/null +++ b/crowdsec-docs/docs/data_sources/http.md @@ -0,0 +1,170 @@ +--- +id: http +title: HTTP +--- + +This module allows the `Security Engine` to acquire logs from an HTTP endpoint. + +## Configuration examples + +To receive logs from an HTTP endpoint with basic auth: +```yaml +source: http +port: 8080 +path: /test +auth_type: basic_auth +basic_auth: + username: test + password: test +labels: + type: mytype +``` + +To receive logs from an HTTP endpoint with headers: +```yaml +source: http +port: 8080 +path: /test +auth_type: headers +headers: + MyHeader: MyValue +labels: + type: mytype +``` + +To receive logs from an HTTP endpoint with TLS and headers: + +```yaml +source: http +port: 8080 +path: /test +auth_type: headers +headers: + MyHeader: MyValue +tls: + server_cert: server.crt + server_key: server.key +labels: + type: mytype +``` + +To receive logs from an HTTP endpoint with mTLS: + +```yaml +source: http +port: 8080 +path: /test +auth_type: mtls +tls: + server_cert: server.crt + server_key: server.key + ca_cert: ca.crt +labels: + type: mytype +``` + +Look at the `configuration parameters` to view all supported options. + +## Parameters + + +### `port` + +The port to listen on. + +Required. + +### `path` + +The endpoint path to listen on. + +:::info +The request method is always `POST`. +::: + +Required. + +### `auth_type` + +The authentication type to use. + +Can be `basic_auth`, `headers`, or `mtls`. + +Required. + +### `basic_auth` + +The basic auth credentials. + +### `basic_auth.username` + +The basic auth username. + +Optional, to use when `auth_type` is `basic_auth`. + +### `basic_auth.password` + +The basic auth password. + +Optional, to use when `auth_type` is `basic_auth`. + +### `headers` + +The headers to send. + +Optional, to use when `auth_type` is `headers`. + +### `tls` + +TLS configuration. + +### `tls.server_cert` + +The server certificate path. + +Optional, to use when `auth_type` is `mtls`. + +### `tls.server_key` + +The server key path. + +Optional, to use when `auth_type` is `mtls`. + +### `tls.ca_cert` + +The CA certificate path. + +Optional, to use when `auth_type` is `mtls`. + +### `custom_status_code` + +The custom status code to return. + +Optional. + +### `custom_headers` + +The custom headers to return. + +Optional. + +### `max_body_size` + +The maximum body size to accept. + +Optional. + +### `timeout` + +The timeout to read the body. + +:::info +The timeout is in duration format, e.g., `5s`. +::: + +Optional. + +## DSN and command-line + +This datasource does not support acquisition from the command line. + From c277c8c61aa1bf3f884e076c5b424048b69951db Mon Sep 17 00:00:00 2001 From: he2ss Date: Fri, 25 Oct 2024 15:41:37 +0200 Subject: [PATCH 2/4] update params --- crowdsec-docs/docs/data_sources/http.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/crowdsec-docs/docs/data_sources/http.md b/crowdsec-docs/docs/data_sources/http.md index 1cbe08fde..c959dc532 100644 --- a/crowdsec-docs/docs/data_sources/http.md +++ b/crowdsec-docs/docs/data_sources/http.md @@ -10,7 +10,7 @@ This module allows the `Security Engine` to acquire logs from an HTTP endpoint. To receive logs from an HTTP endpoint with basic auth: ```yaml source: http -port: 8080 +listen_addr: 127.0.0.1:8080 path: /test auth_type: basic_auth basic_auth: @@ -23,7 +23,7 @@ labels: To receive logs from an HTTP endpoint with headers: ```yaml source: http -port: 8080 +listen_addr: 127.0.0.1:8080 path: /test auth_type: headers headers: @@ -36,7 +36,7 @@ To receive logs from an HTTP endpoint with TLS and headers: ```yaml source: http -port: 8080 +listen_addr: 127.0.0.1:8080 path: /test auth_type: headers headers: @@ -52,7 +52,7 @@ To receive logs from an HTTP endpoint with mTLS: ```yaml source: http -port: 8080 +listen_addr: 127.0.0.1:8080 path: /test auth_type: mtls tls: @@ -68,9 +68,9 @@ Look at the `configuration parameters` to view all supported options. ## Parameters -### `port` +### `listen_addr` -The port to listen on. +The address to listen on (e.g., `1270.0.1:8088`). Required. @@ -82,7 +82,7 @@ The endpoint path to listen on. The request method is always `POST`. ::: -Required. +Optional, default is `/`. ### `auth_type` From d0a8cc7974ee3f9ca2ebf246fc47838ae9fb6e48 Mon Sep 17 00:00:00 2001 From: he2ss Date: Fri, 25 Oct 2024 16:33:20 +0200 Subject: [PATCH 3/4] fix doc for http datasource --- crowdsec-docs/docs/data_sources/http.md | 2 +- .../docs/data_sources/introduction.md | 1 + crowdsec-docs/sidebars.js | 1 + .../version-v1.6.0/data_sources/http.md | 170 ++++++++++++++++++ .../data_sources/introduction.md | 1 + crowdsec-docs/yarn.lock | 2 +- 6 files changed, 175 insertions(+), 2 deletions(-) create mode 100644 crowdsec-docs/versioned_docs/version-v1.6.0/data_sources/http.md diff --git a/crowdsec-docs/docs/data_sources/http.md b/crowdsec-docs/docs/data_sources/http.md index c959dc532..47f94ddca 100644 --- a/crowdsec-docs/docs/data_sources/http.md +++ b/crowdsec-docs/docs/data_sources/http.md @@ -70,7 +70,7 @@ Look at the `configuration parameters` to view all supported options. ### `listen_addr` -The address to listen on (e.g., `1270.0.1:8088`). +The address to listen on (e.g., `127.0.0.1:8088`). Required. diff --git a/crowdsec-docs/docs/data_sources/introduction.md b/crowdsec-docs/docs/data_sources/introduction.md index 6b9373320..2541a21a8 100644 --- a/crowdsec-docs/docs/data_sources/introduction.md +++ b/crowdsec-docs/docs/data_sources/introduction.md @@ -18,6 +18,7 @@ Name | Type | Stream | One-shot [AWS S3](/data_sources/s3.md)| read logs from a S3 bucket | yes | yes [docker](/data_sources/docker.md) | read logs from docker containers | yes | yes [file](/data_sources/file.md) | single files, glob expressions and .gz files | yes | yes +[HTTP](/data_sources/http.md) | read logs from an HTTP endpoint | yes | no [journald](/data_sources/journald.md) | journald via filter | yes | yes [Kafka](/data_sources/kafka.md)| read logs from kafka topic | yes | no [Kubernetes Audit](/data_sources/kubernetes_audit.md) | expose a webhook to receive audit logs from a Kubernetes cluster | yes | no diff --git a/crowdsec-docs/sidebars.js b/crowdsec-docs/sidebars.js index bb18ba3a2..4b0cc0cd7 100644 --- a/crowdsec-docs/sidebars.js +++ b/crowdsec-docs/sidebars.js @@ -131,6 +131,7 @@ "data_sources/s3", "data_sources/docker", "data_sources/file", + "data_sources/http", "data_sources/journald", "data_sources/kafka", "data_sources/kubernetes_audit", diff --git a/crowdsec-docs/versioned_docs/version-v1.6.0/data_sources/http.md b/crowdsec-docs/versioned_docs/version-v1.6.0/data_sources/http.md new file mode 100644 index 000000000..47f94ddca --- /dev/null +++ b/crowdsec-docs/versioned_docs/version-v1.6.0/data_sources/http.md @@ -0,0 +1,170 @@ +--- +id: http +title: HTTP +--- + +This module allows the `Security Engine` to acquire logs from an HTTP endpoint. + +## Configuration examples + +To receive logs from an HTTP endpoint with basic auth: +```yaml +source: http +listen_addr: 127.0.0.1:8080 +path: /test +auth_type: basic_auth +basic_auth: + username: test + password: test +labels: + type: mytype +``` + +To receive logs from an HTTP endpoint with headers: +```yaml +source: http +listen_addr: 127.0.0.1:8080 +path: /test +auth_type: headers +headers: + MyHeader: MyValue +labels: + type: mytype +``` + +To receive logs from an HTTP endpoint with TLS and headers: + +```yaml +source: http +listen_addr: 127.0.0.1:8080 +path: /test +auth_type: headers +headers: + MyHeader: MyValue +tls: + server_cert: server.crt + server_key: server.key +labels: + type: mytype +``` + +To receive logs from an HTTP endpoint with mTLS: + +```yaml +source: http +listen_addr: 127.0.0.1:8080 +path: /test +auth_type: mtls +tls: + server_cert: server.crt + server_key: server.key + ca_cert: ca.crt +labels: + type: mytype +``` + +Look at the `configuration parameters` to view all supported options. + +## Parameters + + +### `listen_addr` + +The address to listen on (e.g., `127.0.0.1:8088`). + +Required. + +### `path` + +The endpoint path to listen on. + +:::info +The request method is always `POST`. +::: + +Optional, default is `/`. + +### `auth_type` + +The authentication type to use. + +Can be `basic_auth`, `headers`, or `mtls`. + +Required. + +### `basic_auth` + +The basic auth credentials. + +### `basic_auth.username` + +The basic auth username. + +Optional, to use when `auth_type` is `basic_auth`. + +### `basic_auth.password` + +The basic auth password. + +Optional, to use when `auth_type` is `basic_auth`. + +### `headers` + +The headers to send. + +Optional, to use when `auth_type` is `headers`. + +### `tls` + +TLS configuration. + +### `tls.server_cert` + +The server certificate path. + +Optional, to use when `auth_type` is `mtls`. + +### `tls.server_key` + +The server key path. + +Optional, to use when `auth_type` is `mtls`. + +### `tls.ca_cert` + +The CA certificate path. + +Optional, to use when `auth_type` is `mtls`. + +### `custom_status_code` + +The custom status code to return. + +Optional. + +### `custom_headers` + +The custom headers to return. + +Optional. + +### `max_body_size` + +The maximum body size to accept. + +Optional. + +### `timeout` + +The timeout to read the body. + +:::info +The timeout is in duration format, e.g., `5s`. +::: + +Optional. + +## DSN and command-line + +This datasource does not support acquisition from the command line. + diff --git a/crowdsec-docs/versioned_docs/version-v1.6.0/data_sources/introduction.md b/crowdsec-docs/versioned_docs/version-v1.6.0/data_sources/introduction.md index 6b9373320..2541a21a8 100644 --- a/crowdsec-docs/versioned_docs/version-v1.6.0/data_sources/introduction.md +++ b/crowdsec-docs/versioned_docs/version-v1.6.0/data_sources/introduction.md @@ -18,6 +18,7 @@ Name | Type | Stream | One-shot [AWS S3](/data_sources/s3.md)| read logs from a S3 bucket | yes | yes [docker](/data_sources/docker.md) | read logs from docker containers | yes | yes [file](/data_sources/file.md) | single files, glob expressions and .gz files | yes | yes +[HTTP](/data_sources/http.md) | read logs from an HTTP endpoint | yes | no [journald](/data_sources/journald.md) | journald via filter | yes | yes [Kafka](/data_sources/kafka.md)| read logs from kafka topic | yes | no [Kubernetes Audit](/data_sources/kubernetes_audit.md) | expose a webhook to receive audit logs from a Kubernetes cluster | yes | no diff --git a/crowdsec-docs/yarn.lock b/crowdsec-docs/yarn.lock index d5048e735..6f9b25bff 100644 --- a/crowdsec-docs/yarn.lock +++ b/crowdsec-docs/yarn.lock @@ -9180,4 +9180,4 @@ yocto-queue@^1.0.0: zwitch@^2.0.0: version "2.0.4" resolved "https://registry.npmjs.org/zwitch/-/zwitch-2.0.4.tgz" - integrity sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A== \ No newline at end of file + integrity sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A== From 3d97f77c1f21ebc57711b5ebbf430865a517ecf1 Mon Sep 17 00:00:00 2001 From: he2ss Date: Tue, 29 Oct 2024 11:23:28 +0100 Subject: [PATCH 4/4] update doc --- crowdsec-docs/docs/data_sources/http.md | 21 +++++++++++++++++++ .../version-v1.6.0/data_sources/http.md | 21 +++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/crowdsec-docs/docs/data_sources/http.md b/crowdsec-docs/docs/data_sources/http.md index 47f94ddca..9ca806a89 100644 --- a/crowdsec-docs/docs/data_sources/http.md +++ b/crowdsec-docs/docs/data_sources/http.md @@ -63,6 +63,27 @@ labels: type: mytype ``` +:::info +If most of cases when the logs are sent in JSON format, you can use the [`transform`](https://docs.crowdsec.net/docs/next/data_sources/intro/#transform) expression to parse the logs. + +For example, if the logs are sent in the following format: +```json +{ + "Records": [ + { + "message": "test", + "timestamp": "2021-01-01T00:00:00Z" + } + ] +} +``` + +the `transform` expression can be: +```yaml +transform: map(JsonExtractSlice(evt.Line.Raw, "Records"), ToJsonString(#)) +``` + + Look at the `configuration parameters` to view all supported options. ## Parameters diff --git a/crowdsec-docs/versioned_docs/version-v1.6.0/data_sources/http.md b/crowdsec-docs/versioned_docs/version-v1.6.0/data_sources/http.md index 47f94ddca..9ca806a89 100644 --- a/crowdsec-docs/versioned_docs/version-v1.6.0/data_sources/http.md +++ b/crowdsec-docs/versioned_docs/version-v1.6.0/data_sources/http.md @@ -63,6 +63,27 @@ labels: type: mytype ``` +:::info +If most of cases when the logs are sent in JSON format, you can use the [`transform`](https://docs.crowdsec.net/docs/next/data_sources/intro/#transform) expression to parse the logs. + +For example, if the logs are sent in the following format: +```json +{ + "Records": [ + { + "message": "test", + "timestamp": "2021-01-01T00:00:00Z" + } + ] +} +``` + +the `transform` expression can be: +```yaml +transform: map(JsonExtractSlice(evt.Line.Raw, "Records"), ToJsonString(#)) +``` + + Look at the `configuration parameters` to view all supported options. ## Parameters