From e02c1a1f9b2a05a6b2fd0e10784c8151e73d6b94 Mon Sep 17 00:00:00 2001 From: jdv Date: Mon, 16 Sep 2024 13:47:56 +0200 Subject: [PATCH 01/15] adding cloudways guide --- .../installation/cloudways.mdx | 283 ++++++++++++++++++ 1 file changed, 283 insertions(+) create mode 100644 crowdsec-docs/unversioned/getting_started/installation/cloudways.mdx diff --git a/crowdsec-docs/unversioned/getting_started/installation/cloudways.mdx b/crowdsec-docs/unversioned/getting_started/installation/cloudways.mdx new file mode 100644 index 000000000..284c520eb --- /dev/null +++ b/crowdsec-docs/unversioned/getting_started/installation/cloudways.mdx @@ -0,0 +1,283 @@ +--- +id: cloudways +title: Cloudways (CrowdSec + WP Remediation) +pagination_prev: getting_started/pre_requisites +pagination_next: getting_started/next_steps +--- + +@import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import CodeBlock from '@theme/CodeBlock'; + +# Preamble +Cloudways is a managed cloud hosting platform that simplifies the process of hosting websites and applications on various cloud providers. +It provides you with a SSH access but with limited rights. +**However** there is a way to run CrowdSec on Cloudways and get both behavior detection on your services (nginx + apache) and applying remediation with our [WP plugin](/u/bouncers/wordpress.mdx) also unlocking the blocklist feature. +This guide is a bit longer than the other ones as it describes all specific steps needed for Cloudways integration. + +We'll guide you through the following steps: +1. [Install CrowdSec from the static build](#install-crowdsec-from-the-static-build) +2. [Setup acquisitions and detection collections](#setup-acquisitions-and-detection-collections) +3. [Run a behavior detection on your past logs to see what it would have found](#run-a-behavior-detection-on-your-past-logs-to-see-what-it-would-have-found) +4. [Make CrowdSec run as a service at user level](#make-crowdsec-service-run-at-user-level) +5. [Bind it to the WP plugin to block the detected attackers](#bind-it-to-the-wp-plugin-to-block-the-detected-attackers) + +## Install CrowdSec from the static build +In this section, we'll get the latest static build of CrowdSec, build the folder hierarchy with the slightly tweaked test_env script and create the necessary config for the Local API and Central API. + +### Setup CrowdSec static build +> For this setup we'll put CrowdSec in the */home/master/crowdsec* folder. +#### Get the static build +- Go to https://github.com/crowdsecurity/crowdsec/releases +- Choose the version you want (at the time of writing 1.6.3 was the latest release) +- Scroll down past the changelog, in the **Assets** section copy the link to the **crowdsec-release.tgz** file +- download it in your */home/master* folder, example: +```bash +wget https://github.com/crowdsecurity/crowdsec/releases/download/v1.6.3/crowdsec-release.tgz +``` +- Extract the archive: +```bash +tar -xvzf crowdsec-release.tgz +``` +- Rename the extracted folder to *crowdsec*: +```bash +mv crowdsec-v1.6.3 crowdsec +``` +#### Create the folder hierarchy +- cd into the *crowdsec* folder: +```bash +cd crowdsec +``` +- Tweak the test_env script to create the necessary folders and config: +```bash +sed -i 's|BASE="./tests"|BASE="./"|' test_env.sh +``` +- Run the script: +```bash +./test_env.sh +``` +- Check one config file symlink to make sure the tweak worked: +```bash +ls -la config/parsers/s00-raw/syslog-logs.yaml +``` +Should output *config/parsers/s00-raw/syslog-logs.yaml -> /home/master/crowdsec/config/hub/parsers/s00-raw/crowdsecurity/syslog-logs.yaml* + +#### Create the config +We'll take the template config, update a few ports to avoid conflicts and setup the Local API and Central API. +- We'll use the dev.yml template to create our config.yaml: +```bash +rn dev.yml > config.yaml +``` +- Now lets update the port number for the Local API. +- Open the config.yaml file in you editor of choice and change the following values: + - common section: + - change **log_media: file** + - add **log_dir: ./logs/** + - api/server section: + - listen on a free port, example 19443 + - listen_uri:127.0.0.1:19443 + +#### Init/Reset CAPI and LAPI credentials + - Quickly register on CAPI. This will create the necessary credentials in the *config/online_api_credentials.yaml* file +```bash +./cscli -c config.yaml capi register +``` + - It should tell you to restart CrowdSec, ignore it for now, we'll do it later. + - LAPI setup relies on "machines" + - Remove the existing machine and create a new one in auto: +```bash +./cscli -c config.yaml machines list //ignore the warning it's normal for now +``` + - You should see something like this +```bash +──────────────────────────────────────────────────────────────────────────────────────── + Name IP Address Last Update Status Version OS Auth Type Last Heartbeat +──────────────────────────────────────────────────────────────────────────────────────── + test 2024-09-12T10:04:52Z ✔️ ? password ⚠️ - +──────────────────────────────────────────────────────────────────────────────────────── +``` + - Delete the test machine +```bash +./cscli -c config.yaml machines delete test_env +``` + - Create a new default one with --force to override the existing credentials file +```bash +./cscli machines add my_logprocessor --auto --force +``` + - C that the credential file has the proper port : *cat ./config/local_api_credentials.yaml* +```yaml +url: http://127.0.0.1 +login: my_logprocessor +password: 321QSd54QERG321sq54AZEqs45AZDQSd654z65fps +``` + +## Setup acquisitions and detection collections +Acquisition configuration indicates to CrowdSec what log files it should look at. +The Detection collections include parsers config and bad behavior detection scenarios for given services. + +In our case we'll look at the nginx logs and apache2 logs. +- Identify the name of your application folder: ls /home/master/applications +- There should be a folder in there, lets say "abcdefghij" +- We'll replace the content of the config/acquis.yaml file (with you editor of choice) with the following: +```yaml +filenames: + - /home/master/applications/abcdefghij/logs/nginx_*.log +labels: + type: nginx +--- +filenames: + - /home/master/applications/abcdefghij/logs/apache_*.log +labels: + type: apache2 +``` +- Don't forget to put the appropriate path to your logs and not "abcdefghij" + +### Getting collections +Now we'll install the collections for nginx and apache2. +You can find our catalog on our [Hub](https://hub.crowdsec.net). +- Run the following command to install the collections: +```bash +./cscli -c config.yaml collections install crowdsecurity/nginx crowdsecurity/apache2 +``` +### Making the collections auto update +CrowdSec collection often get updated with the behavior detections. +CrowdSec teams create and currate community scenarios allowing its users to benefit from the latest vulnerabilities detection. +We'll allow hub auto-update with a cron: + - Create a hub_update.sh file in the crowdsec folder: +```bash +#!/bin/sh + +test -x /home/master/crowdsec/cscli || exit 0 + +# splay hub upgrade and crowdsec reload +sleep "$(seq 1 300 | shuf -n 1)" + +/home/master/crowdsec/cscli -c /home/master/crowdsec/config.yaml --error hub update + +upgraded=$(/home/master/crowdsec/cscli -c /home/master/crowdsec/config.yaml --error hub upgrade) +if [ -n "$upgraded" ]; then + systemctl --user reload crowdsec +fi + +exit 0 +``` + - Add it to crontab, every day at 6 for example +``` +0 6 * * * /home/master/crowdsec/hub_update.sh +``` + +## Run a behavior detection on your past logs to see what it would have found +We can run the behavior detection on the past logs to catch alerts that happened in the past. +We'll run it on the nginx access logs and the first archive of nginx access logs (previous day) +- Run the behavior detection on the past logs: +```bash +./crowdsec -c config.yaml -dsn file:///home/master/applications/abcdefghij/logs/nginx_*.access.log --type nginx --no-api +``` +- Again, dont forget to put your own application folder and not "abcdefghij" +- Note that **dsn** parameter take the **file://***/ protocol and an **absolute path** +- After you ran the detection, detected alerts should be listed in: +```bash +./cscli -c config.yaml alerts list +``` + +## Make CrowdSec service run at user level +We want CrowdSec to run in the background and start at boot. +For this we'll add a systemd service in the user level. + +### Create the systemd service for user +- At the time of writting (for v1.6.3) you can use the following content: +- Create and edit ~/.config/systemd/user/crowdsec.service +```bash +[Unit] +Description=Crowdsec agent + +[Service] +WorkingDirectory=/home/master/crowdsec +Type=notify +Environment=LC_ALL=C LANG=C +ExecStartPre=/home/master/crowdsec/crowdsec -c /home/master/crowdsec/config.yaml -t -error +ExecStart=/home/master/crowdsec/crowdsec -c /home/master/crowdsec/config.yaml +#ExecStartPost=/bin/sleep 0.1 +ExecReload=/home/master/crowdsec/crowdsec -c /home/master/crowdsec/config.yaml -t -error +ExecReload=/bin/kill -HUP $MAINPID +Restart=always +RestartSec=60 + +[Install] +WantedBy=multi-user.target +``` +- Note that if you want to do it yourself the process is: + - Get the service description file from https://github.com/crowdsecurity/crowdsec/blob/master/config/crowdsec.service + - Move it to the user systemd user folder + - Modify this file to have the proper path to crowdsec executable and config + +### Enable the service to run at boot +For a user level process to keep running after you close the connection we need to activate the "linger" + - Run the following command: +```bash +loginctl enable-linger +``` + - Then have systemctl reload and run crowdsec +```bash +systemctl --user daemon-reload +systemctl --user enable --now crowdsec +``` + - Check the status of the service +```bash +systemctl --user status crowdsec +``` + - In the future you can **systemctl --user start crowdsec** or stop or restart + +### Checking that CrowdSec works +We ran a behavior detection on the past logs so we might already have acquisition and parsing metrics. +But to check that its working, you can visit your website + - It should generate lines of logs + - As soon as new log lines arrive in any of those: + - You should see the acquisition metrics appear/update + - And the resulting parser acquisition and metrics +```bash +./cscli metrics -c config.yaml +``` +- looking something like +```bash +Acquisition Metrics: +╭──────────────────────────────────────────────────────────────────────────────────────────────────────────────┬────────────┬──────────────┬────────────────┬────────────────────────┬───────────────────╮ +│ Source │ Lines read │ Lines parsed │ Lines unparsed │ Lines poured to bucket │ Lines whitelisted │ +├──────────────────────────────────────────────────────────────────────────────────────────────────────────────┼────────────┼──────────────┼────────────────┼────────────────────────┼───────────────────┤ +│ file:/home/master/applications/abcdefghij/logs/apache_wordpress-1211499-4678369.cloudwaysapps.com.access.log │ 1 │ 1 │ - │ - │ - │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────┴────────────┴──────────────┴────────────────┴────────────────────────┴───────────────────╯ + +[...] + +Parser Metrics: +╭──────────────────────────────────┬──────┬────────┬──────────╮ +│ Parsers │ Hits │ Parsed │ Unparsed │ +├──────────────────────────────────┼──────┼────────┼──────────┤ +│ child-crowdsecurity/apache2-logs │ 1 │ 1 │ - │ +│ child-crowdsecurity/http-logs │ 3 │ 3 │ - │ +│ crowdsecurity/apache2-logs │ 1 │ 1 │ - │ +│ crowdsecurity/dateparse-enrich │ 1 │ 1 │ - │ +│ crowdsecurity/geoip-enrich │ 1 │ 1 │ - │ +│ crowdsecurity/http-logs │ 1 │ 1 │ - │ +│ crowdsecurity/non-syslog │ 1 │ 1 │ - │ +╰──────────────────────────────────┴──────┴────────┴──────────╯ + +``` + +## Bind it to the WP plugin to block the detected attackers +Now that we have CrowdSec running and detecting bad behaviors. +Alerts are raised and decisions to block bad actors are stored in the local DB. +To actually apply a remediation and ban the attackers from your website you need: +- To create a bouncer API key: +```bash +./cscli -c config.yaml bouncers add my_wp_bouncer +``` +- You should see something like this: +```bash +API key for 'my_wp_bouncer': + + OI8BQQqMcasoeuxK2g5lMSHPLVkH1tARqLIW0HS3cIY + +Please keep this key since you will not be able to retrieve it! +``` +- Add those credentials to your WP bouncer plugin as described in the [WP plugin documentation](/u/bouncers/wordpress.mdx#configurations) From 304d39fcb62f48d4a184840569fa970cf24588fc Mon Sep 17 00:00:00 2001 From: jdv Date: Mon, 16 Sep 2024 14:07:30 +0200 Subject: [PATCH 02/15] adding logo + menu ++ --- crowdsec-docs/sidebarsUnversioned.js | 1 + crowdsec-docs/src/components/QuickStart.js | 6 ++++++ crowdsec-docs/static/img/logo-cloudways.svg | 2 ++ .../unversioned/getting_started/installation/cloudways.mdx | 2 +- 4 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 crowdsec-docs/static/img/logo-cloudways.svg diff --git a/crowdsec-docs/sidebarsUnversioned.js b/crowdsec-docs/sidebarsUnversioned.js index 7cda202f4..cc95731a4 100644 --- a/crowdsec-docs/sidebarsUnversioned.js +++ b/crowdsec-docs/sidebarsUnversioned.js @@ -492,6 +492,7 @@ module.exports = { "getting_started/installation/pfsense", "getting_started/installation/opnsense", "getting_started/installation/whm", + "getting_started/installation/cloudways", ], }, { diff --git a/crowdsec-docs/src/components/QuickStart.js b/crowdsec-docs/src/components/QuickStart.js index 4c1853365..03c8c7ab6 100644 --- a/crowdsec-docs/src/components/QuickStart.js +++ b/crowdsec-docs/src/components/QuickStart.js @@ -10,6 +10,7 @@ import { import opnsenseLogo from "@site/static/img/logo-opnsense.svg"; import pfSenseLogo from "@site/static/img/logo-pfsense.svg"; import whmLogo from "@site/static/img/logo-whm.svg"; +import cloudwaysLogo from "@site/static/img/logo-cloudways.svg"; import Link from "@docusaurus/Link"; const staticData = [ @@ -58,6 +59,11 @@ const staticData = [ text: "WHM", link: "/u/getting_started/installation/whm", }, + { + icon: cloudwaysLogo, + text: "Cloudways", + link: "/u/getting_started/installation/cloudways", + }, ]; export default function QuickStart() { diff --git a/crowdsec-docs/static/img/logo-cloudways.svg b/crowdsec-docs/static/img/logo-cloudways.svg new file mode 100644 index 000000000..cc6e5dc8c --- /dev/null +++ b/crowdsec-docs/static/img/logo-cloudways.svg @@ -0,0 +1,2 @@ + + \ No newline at end of file diff --git a/crowdsec-docs/unversioned/getting_started/installation/cloudways.mdx b/crowdsec-docs/unversioned/getting_started/installation/cloudways.mdx index 284c520eb..e43c3f6c8 100644 --- a/crowdsec-docs/unversioned/getting_started/installation/cloudways.mdx +++ b/crowdsec-docs/unversioned/getting_started/installation/cloudways.mdx @@ -5,7 +5,7 @@ pagination_prev: getting_started/pre_requisites pagination_next: getting_started/next_steps --- -@import Tabs from '@theme/Tabs'; +import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; import CodeBlock from '@theme/CodeBlock'; From da72d288720bc791b6370da2b422339336dcc3b9 Mon Sep 17 00:00:00 2001 From: Laurence Date: Wed, 25 Sep 2024 09:37:52 +0100 Subject: [PATCH 03/15] enhance: wip updates --- .../installation/cloudways.mdx | 83 ++++++++++++------- 1 file changed, 51 insertions(+), 32 deletions(-) diff --git a/crowdsec-docs/unversioned/getting_started/installation/cloudways.mdx b/crowdsec-docs/unversioned/getting_started/installation/cloudways.mdx index e43c3f6c8..fe5d77659 100644 --- a/crowdsec-docs/unversioned/getting_started/installation/cloudways.mdx +++ b/crowdsec-docs/unversioned/getting_started/installation/cloudways.mdx @@ -9,28 +9,33 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; import CodeBlock from '@theme/CodeBlock'; -# Preamble -Cloudways is a managed cloud hosting platform that simplifies the process of hosting websites and applications on various cloud providers. -It provides you with a SSH access but with limited rights. -**However** there is a way to run CrowdSec on Cloudways and get both behavior detection on your services (nginx + apache) and applying remediation with our [WP plugin](/u/bouncers/wordpress.mdx) also unlocking the blocklist feature. -This guide is a bit longer than the other ones as it describes all specific steps needed for Cloudways integration. - -We'll guide you through the following steps: -1. [Install CrowdSec from the static build](#install-crowdsec-from-the-static-build) -2. [Setup acquisitions and detection collections](#setup-acquisitions-and-detection-collections) -3. [Run a behavior detection on your past logs to see what it would have found](#run-a-behavior-detection-on-your-past-logs-to-see-what-it-would-have-found) -4. [Make CrowdSec run as a service at user level](#make-crowdsec-service-run-at-user-level) -5. [Bind it to the WP plugin to block the detected attackers](#bind-it-to-the-wp-plugin-to-block-the-detected-attackers) +# Preface +Cloudways is a managed cloud hosting platform that makes it easy to host websites and applications across different cloud providers. + +It gives you SSH access, but with limited permissions. + +**However**, you can still run the [Security Engine](/docs/next/intro) on Cloudways to enable behavior detection for your services (Nginx and Apache) and use our [WordPress plugin](bouncers/wordpress.mdx) to apply remediation, including the blocklist feature. This guide is more detailed than others because it covers the specific steps required for integrating CrowdSec with Cloudways. + +We'll walk you through the following steps: +1. [Install CrowdSec using the static build](#install-crowdsec-from-the-static-build) +2. [Set up acquisition and detection collections](#setup-acquisitions-and-detection-collections) +3. [Run behavior detection on your past logs to see what would have been flagged](#run-a-behavior-detection-on-your-past-logs-to-see-what-it-would-have-found) +4. [Make CrowdSec run as a user-level service](#make-crowdsec-service-run-at-user-level) +5. [Connect it to the WordPress plugin to block detected attackers](#bind-it-to-the-wp-plugin-to-block-the-detected-attackers) ## Install CrowdSec from the static build In this section, we'll get the latest static build of CrowdSec, build the folder hierarchy with the slightly tweaked test_env script and create the necessary config for the Local API and Central API. ### Setup CrowdSec static build > For this setup we'll put CrowdSec in the */home/master/crowdsec* folder. + #### Get the static build -- Go to https://github.com/crowdsecurity/crowdsec/releases -- Choose the version you want (at the time of writing 1.6.3 was the latest release) +- Go to the [latest release page](https://github.com/crowdsecurity/crowdsec/releases/latest) - Scroll down past the changelog, in the **Assets** section copy the link to the **crowdsec-release.tgz** file +- ensure you are within the `/home/master` folder +```bash +cd /home/master +``` - download it in your */home/master* folder, example: ```bash wget https://github.com/crowdsecurity/crowdsec/releases/download/v1.6.3/crowdsec-release.tgz @@ -39,7 +44,7 @@ wget https://github.com/crowdsecurity/crowdsec/releases/download/v1.6.3/crowdsec ```bash tar -xvzf crowdsec-release.tgz ``` -- Rename the extracted folder to *crowdsec*: +- Rename the extracted folder to *crowdsec*: ```bash mv crowdsec-v1.6.3 crowdsec ``` @@ -50,7 +55,7 @@ cd crowdsec ``` - Tweak the test_env script to create the necessary folders and config: ```bash -sed -i 's|BASE="./tests"|BASE="./"|' test_env.sh +sed -i 's|BASE="./tests"|BASE="/home/master/crowdsec"|' test_env.sh ``` - Run the script: ```bash @@ -66,24 +71,38 @@ Should output *config/parsers/s00-raw/syslog-logs.yaml -> /home/master/crowdsec/ We'll take the template config, update a few ports to avoid conflicts and setup the Local API and Central API. - We'll use the dev.yml template to create our config.yaml: ```bash -rn dev.yml > config.yaml +mv dev.yml config.yaml ``` -- Now lets update the port number for the Local API. -- Open the config.yaml file in you editor of choice and change the following values: - - common section: - - change **log_media: file** - - add **log_dir: ./logs/** - - api/server section: - - listen on a free port, example 19443 - - listen_uri:127.0.0.1:19443 - -#### Init/Reset CAPI and LAPI credentials - - Quickly register on CAPI. This will create the necessary credentials in the *config/online_api_credentials.yaml* file + +#### Update the configuration +> /home/master/crowdsec/config.yaml + +We need to make alterations to the config file to avoid conflicts and setup the Local API. Open the `/home/master/crowdsec/config.yaml` with your favourite terminal editor and make the following changes: + +```yaml +common: + log_media: file ## Alter this line from stdout to file + log_dir: ./logs ## Add this line, ensure it indented correctly +api: + server: + listen_uri: 127.0.0.1:8081 ## Change this line to a free port like 19443 +``` + +#### Init CAPI (Central API) credentials + +We can initilized the CAPI credentials with the following command: + ```bash ./cscli -c config.yaml capi register ``` - - It should tell you to restart CrowdSec, ignore it for now, we'll do it later. - - LAPI setup relies on "machines" + +This will generate `/home/master/crowdsec/config/online_api_credentials.yaml` make sure to keep this file safe. + +:::warning +The output will instruct you to restart the service, but we'll do that later. +::: + +#### Reset LAPI (Local API) credentials - Remove the existing machine and create a new one in auto: ```bash ./cscli -c config.yaml machines list //ignore the warning it's normal for now @@ -121,12 +140,12 @@ In our case we'll look at the nginx logs and apache2 logs. - We'll replace the content of the config/acquis.yaml file (with you editor of choice) with the following: ```yaml filenames: - - /home/master/applications/abcdefghij/logs/nginx_*.log + - /home/master/applications/**/logs/nginx_*.log labels: type: nginx --- filenames: - - /home/master/applications/abcdefghij/logs/apache_*.log + - /home/master/applications/**/logs/apache_*.log labels: type: apache2 ``` From 6a06530f74970c3320082cee54537fb0beedc142 Mon Sep 17 00:00:00 2001 From: jdv Date: Thu, 26 Sep 2024 11:47:11 +0200 Subject: [PATCH 04/15] tweaks on config --- .../installation/cloudways.mdx | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/crowdsec-docs/unversioned/getting_started/installation/cloudways.mdx b/crowdsec-docs/unversioned/getting_started/installation/cloudways.mdx index fe5d77659..cd4dd1fec 100644 --- a/crowdsec-docs/unversioned/getting_started/installation/cloudways.mdx +++ b/crowdsec-docs/unversioned/getting_started/installation/cloudways.mdx @@ -77,15 +77,30 @@ mv dev.yml config.yaml #### Update the configuration > /home/master/crowdsec/config.yaml -We need to make alterations to the config file to avoid conflicts and setup the Local API. Open the `/home/master/crowdsec/config.yaml` with your favourite terminal editor and make the following changes: +We need to make alterations to the config file for this static install and to avoid conflicts and setup the Local API. +- First, replace all* ./ *with* /home/master/crowdsec/ *in the config.yaml file +```bash +sed -i 's|./|/home/master/crowdsec/|g' config.yaml +``` +Open the `/home/master/crowdsec/config.yaml` with your favourite terminal editor and make the following changes: +- Update log_media and add log dir ```yaml common: log_media: file ## Alter this line from stdout to file log_dir: ./logs ## Add this line, ensure it indented correctly +``` +- Then uncomment and replace the hubdir with the correct path: +```yaml +[...] +hub_dir: /home/master/crowdsec/config/hub +``` +- Finally, change the local API port to 19443 in order to avoid conflicts +```yaml +[...] api: server: - listen_uri: 127.0.0.1:8081 ## Change this line to a free port like 19443 + listen_uri: 127.0.0.1:19443 ``` #### Init CAPI (Central API) credentials From 2037b006981308a0992e54c54122a3ff7523c30a Mon Sep 17 00:00:00 2001 From: jdv Date: Thu, 26 Sep 2024 14:09:32 +0200 Subject: [PATCH 05/15] adding aliases --- .../installation/cloudways.mdx | 30 ++++++++++++++----- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/crowdsec-docs/unversioned/getting_started/installation/cloudways.mdx b/crowdsec-docs/unversioned/getting_started/installation/cloudways.mdx index cd4dd1fec..418b4bf9a 100644 --- a/crowdsec-docs/unversioned/getting_started/installation/cloudways.mdx +++ b/crowdsec-docs/unversioned/getting_started/installation/cloudways.mdx @@ -102,13 +102,27 @@ api: server: listen_uri: 127.0.0.1:19443 ``` +#### Create some quality of life aliases +In order to make the command line easier to use, we'll create some aliases for the CrowdSec CLI and crowdsec itself. +This way you won't have to call it from the full path with the config param each time. + +- Add the following to your* /home/master/.bash_aliases* file: +```bash +alias cscli="/home/master/crowdsec/cscli -c /home/master/crowdsec/config.yaml" +alias crowdsec="/home/master/crowdsec/crowdsec -c /home/master/crowdsec/config.yaml" +``` + +- Reload your bash profile: +```bash +source /home/master/.bashrc +``` #### Init CAPI (Central API) credentials We can initilized the CAPI credentials with the following command: ```bash -./cscli -c config.yaml capi register +cscli capi register ``` This will generate `/home/master/crowdsec/config/online_api_credentials.yaml` make sure to keep this file safe. @@ -120,7 +134,7 @@ The output will instruct you to restart the service, but we'll do that later. #### Reset LAPI (Local API) credentials - Remove the existing machine and create a new one in auto: ```bash -./cscli -c config.yaml machines list //ignore the warning it's normal for now +cscli machines list //ignore the warning it's normal for now ``` - You should see something like this ```bash @@ -132,11 +146,11 @@ The output will instruct you to restart the service, but we'll do that later. ``` - Delete the test machine ```bash -./cscli -c config.yaml machines delete test_env +cscli machines delete test_env ``` - Create a new default one with --force to override the existing credentials file ```bash -./cscli machines add my_logprocessor --auto --force +cscli machines add my_logprocessor --auto --force ``` - C that the credential file has the proper port : *cat ./config/local_api_credentials.yaml* ```yaml @@ -171,7 +185,7 @@ Now we'll install the collections for nginx and apache2. You can find our catalog on our [Hub](https://hub.crowdsec.net). - Run the following command to install the collections: ```bash -./cscli -c config.yaml collections install crowdsecurity/nginx crowdsecurity/apache2 +cscli collections install crowdsecurity/nginx crowdsecurity/apache2 ``` ### Making the collections auto update CrowdSec collection often get updated with the behavior detections. @@ -211,7 +225,7 @@ We'll run it on the nginx access logs and the first archive of nginx access logs - Note that **dsn** parameter take the **file://***/ protocol and an **absolute path** - After you ran the detection, detected alerts should be listed in: ```bash -./cscli -c config.yaml alerts list +cscli alerts list ``` ## Make CrowdSec service run at user level @@ -270,7 +284,7 @@ But to check that its working, you can visit your website - You should see the acquisition metrics appear/update - And the resulting parser acquisition and metrics ```bash -./cscli metrics -c config.yaml +cscli metrics -c config.yaml ``` - looking something like ```bash @@ -304,7 +318,7 @@ Alerts are raised and decisions to block bad actors are stored in the local DB. To actually apply a remediation and ban the attackers from your website you need: - To create a bouncer API key: ```bash -./cscli -c config.yaml bouncers add my_wp_bouncer +cscli bouncers add my_wp_bouncer ``` - You should see something like this: ```bash From 610dd51be3b6223cadf60934f47cba5db1ffd37b Mon Sep 17 00:00:00 2001 From: jdv Date: Thu, 17 Oct 2024 17:46:39 +0200 Subject: [PATCH 06/15] rebased and fixed app names replacing by ** --- .../getting_started/installation/cloudways.mdx | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/crowdsec-docs/unversioned/getting_started/installation/cloudways.mdx b/crowdsec-docs/unversioned/getting_started/installation/cloudways.mdx index 418b4bf9a..e2d6c24fb 100644 --- a/crowdsec-docs/unversioned/getting_started/installation/cloudways.mdx +++ b/crowdsec-docs/unversioned/getting_started/installation/cloudways.mdx @@ -164,9 +164,8 @@ Acquisition configuration indicates to CrowdSec what log files it should look at The Detection collections include parsers config and bad behavior detection scenarios for given services. In our case we'll look at the nginx logs and apache2 logs. -- Identify the name of your application folder: ls /home/master/applications -- There should be a folder in there, lets say "abcdefghij" -- We'll replace the content of the config/acquis.yaml file (with you editor of choice) with the following: +- We'll use wildcards to work with any application name of your application folder: ls /home/master/applications +- Replace the content of the config/acquis.yaml file (with you editor of choice) with the following: ```yaml filenames: - /home/master/applications/**/logs/nginx_*.log @@ -178,7 +177,6 @@ filenames: labels: type: apache2 ``` -- Don't forget to put the appropriate path to your logs and not "abcdefghij" ### Getting collections Now we'll install the collections for nginx and apache2. @@ -219,9 +217,8 @@ We can run the behavior detection on the past logs to catch alerts that happened We'll run it on the nginx access logs and the first archive of nginx access logs (previous day) - Run the behavior detection on the past logs: ```bash -./crowdsec -c config.yaml -dsn file:///home/master/applications/abcdefghij/logs/nginx_*.access.log --type nginx --no-api +./crowdsec -c config.yaml -dsn file:///home/master/applications/\*\*/logs/nginx_*.access.log --type nginx --no-api ``` -- Again, dont forget to put your own application folder and not "abcdefghij" - Note that **dsn** parameter take the **file://***/ protocol and an **absolute path** - After you ran the detection, detected alerts should be listed in: ```bash From 59ea577ce00e4e22e649de3c21f2ab2ef733ead5 Mon Sep 17 00:00:00 2001 From: jdv Date: Thu, 7 Nov 2024 17:57:58 +0100 Subject: [PATCH 07/15] addition of log rotation fix script and cron --- .../installation/cloudways.mdx | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/crowdsec-docs/unversioned/getting_started/installation/cloudways.mdx b/crowdsec-docs/unversioned/getting_started/installation/cloudways.mdx index e2d6c24fb..eb34c04f1 100644 --- a/crowdsec-docs/unversioned/getting_started/installation/cloudways.mdx +++ b/crowdsec-docs/unversioned/getting_started/installation/cloudways.mdx @@ -211,6 +211,49 @@ exit 0 ``` 0 6 * * * /home/master/crowdsec/hub_update.sh ``` +### Make sure log rotation not breaking acquisition +As CrowdSec is not running as root in our current context, there could be some race conditions with log rotation file creation making the acquisition fail. +Future versions of CrowdSec might address this issue, but for now, we can use a simple script to ensure the acquisition is not broken. + +- Create a script to ensure the acquisition is not broken + +```bash +vi /home/master/crowdsec/check_rotation.sh +``` + +```bash +#!/bin/bash + +# Set the path to your CrowdSec log file +LOG_FILE="/home/master/crowdsec/logs/crowdsec.log" + +# Get today's date in the format used in the logs (UTC time) +TODAY=$(date -u +"%Y-%m-%d") + +# Define the error pattern to search for +ERROR_PATTERN='level=warning .* died : Unable to open file .*: permission denied' + +# Search for the error in today's logs +if grep "$TODAY" "$LOG_FILE" | grep -qE "$ERROR_PATTERN"; then + # Log the action + echo "$(date): Error found, restarting CrowdSec service" >> /home/master/crowdsec/logs/crowdsec_rotation_fail.log + + # Restart the CrowdSec service + systemctl restart --user crowdsec + + # Log the completion + echo "$(date): CrowdSec service restarted successfully" >> /home/master/crowdsec/logs/crowdsec_rotation_fail.log +else + # Log that no action was taken + echo "$(date): No error found, no action taken" >> /home/master/crowdsec/logs/crowdsec_rotation_fail.log +fi +``` + +Make the check run every day at 00:01 +```bash +1 0 * * * /home/master/crowdsec/check_rotation.sh +``` + ## Run a behavior detection on your past logs to see what it would have found We can run the behavior detection on the past logs to catch alerts that happened in the past. From 9dd1ff76dc37b060cf4fd314963f65f7cd14adce Mon Sep 17 00:00:00 2001 From: jdv Date: Thu, 14 Nov 2024 11:33:09 +0100 Subject: [PATCH 08/15] nano change bash to sh --- .../unversioned/getting_started/installation/cloudways.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crowdsec-docs/unversioned/getting_started/installation/cloudways.mdx b/crowdsec-docs/unversioned/getting_started/installation/cloudways.mdx index eb34c04f1..9ea34cf1a 100644 --- a/crowdsec-docs/unversioned/getting_started/installation/cloudways.mdx +++ b/crowdsec-docs/unversioned/getting_started/installation/cloudways.mdx @@ -222,7 +222,7 @@ vi /home/master/crowdsec/check_rotation.sh ``` ```bash -#!/bin/bash +#!/bin/sh # Set the path to your CrowdSec log file LOG_FILE="/home/master/crowdsec/logs/crowdsec.log" From 88efbb8ff17560b0911c3746d2dfd6d65f139f9d Mon Sep 17 00:00:00 2001 From: jdv Date: Thu, 14 Nov 2024 14:49:18 +0100 Subject: [PATCH 09/15] forgot the sh for cron comands --- .../unversioned/getting_started/installation/cloudways.mdx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crowdsec-docs/unversioned/getting_started/installation/cloudways.mdx b/crowdsec-docs/unversioned/getting_started/installation/cloudways.mdx index 9ea34cf1a..bf3e07a35 100644 --- a/crowdsec-docs/unversioned/getting_started/installation/cloudways.mdx +++ b/crowdsec-docs/unversioned/getting_started/installation/cloudways.mdx @@ -209,7 +209,7 @@ exit 0 ``` - Add it to crontab, every day at 6 for example ``` -0 6 * * * /home/master/crowdsec/hub_update.sh +0 6 * * * sh /home/master/crowdsec/hub_update.sh ``` ### Make sure log rotation not breaking acquisition As CrowdSec is not running as root in our current context, there could be some race conditions with log rotation file creation making the acquisition fail. @@ -251,7 +251,7 @@ fi Make the check run every day at 00:01 ```bash -1 0 * * * /home/master/crowdsec/check_rotation.sh +1 0 * * * sh /home/master/crowdsec/check_rotation.sh ``` From 50f609492efbae6759c1a10a864d96aae4d66390 Mon Sep 17 00:00:00 2001 From: jdv Date: Tue, 19 Nov 2024 15:25:13 +0100 Subject: [PATCH 10/15] added env twek to cron task trick --- .../unversioned/getting_started/installation/cloudways.mdx | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crowdsec-docs/unversioned/getting_started/installation/cloudways.mdx b/crowdsec-docs/unversioned/getting_started/installation/cloudways.mdx index bf3e07a35..5d9e732de 100644 --- a/crowdsec-docs/unversioned/getting_started/installation/cloudways.mdx +++ b/crowdsec-docs/unversioned/getting_started/installation/cloudways.mdx @@ -238,6 +238,9 @@ if grep "$TODAY" "$LOG_FILE" | grep -qE "$ERROR_PATTERN"; then # Log the action echo "$(date): Error found, restarting CrowdSec service" >> /home/master/crowdsec/logs/crowdsec_rotation_fail.log + # Providing env context to the cron job + export XDG_RUNTIME_DIR=/run/user/$(id -u) + export DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/$(id -u)/bus # Restart the CrowdSec service systemctl restart --user crowdsec From 4af5edec8a882396896afb5218f64c8dd16a3854 Mon Sep 17 00:00:00 2001 From: jdv Date: Thu, 21 Nov 2024 14:31:27 +0100 Subject: [PATCH 11/15] added env tweak to cron task trick to hub update too --- .../unversioned/getting_started/installation/cloudways.mdx | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crowdsec-docs/unversioned/getting_started/installation/cloudways.mdx b/crowdsec-docs/unversioned/getting_started/installation/cloudways.mdx index 5d9e732de..51e524666 100644 --- a/crowdsec-docs/unversioned/getting_started/installation/cloudways.mdx +++ b/crowdsec-docs/unversioned/getting_started/installation/cloudways.mdx @@ -202,6 +202,9 @@ sleep "$(seq 1 300 | shuf -n 1)" upgraded=$(/home/master/crowdsec/cscli -c /home/master/crowdsec/config.yaml --error hub upgrade) if [ -n "$upgraded" ]; then + # Providing env context to the cron job + export XDG_RUNTIME_DIR=/run/user/$(id -u) + export DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/$(id -u)/bus systemctl --user reload crowdsec fi From 08e6b8bc74946986ae3310fd9464f901728ea7a6 Mon Sep 17 00:00:00 2001 From: jdv Date: Thu, 21 Nov 2024 15:04:46 +0100 Subject: [PATCH 12/15] mini typo --- .../unversioned/getting_started/installation/cloudways.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crowdsec-docs/unversioned/getting_started/installation/cloudways.mdx b/crowdsec-docs/unversioned/getting_started/installation/cloudways.mdx index 51e524666..489a23c75 100644 --- a/crowdsec-docs/unversioned/getting_started/installation/cloudways.mdx +++ b/crowdsec-docs/unversioned/getting_started/installation/cloudways.mdx @@ -152,7 +152,7 @@ cscli machines delete test_env ```bash cscli machines add my_logprocessor --auto --force ``` - - C that the credential file has the proper port : *cat ./config/local_api_credentials.yaml* + - Check that the credential file has the proper port : *cat ./config/local_api_credentials.yaml* ```yaml url: http://127.0.0.1 login: my_logprocessor From 7725f40bdf9936fea5539a43fc7c1575fd172774 Mon Sep 17 00:00:00 2001 From: jdv Date: Mon, 6 Jan 2025 10:01:02 +0100 Subject: [PATCH 13/15] hidding links to article and labeling it as experimental --- crowdsec-docs/sidebarsUnversioned.js | 1 - crowdsec-docs/src/components/QuickStart.js | 11 ++++++----- .../getting_started/installation/cloudways.mdx | 7 ++++++- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/crowdsec-docs/sidebarsUnversioned.js b/crowdsec-docs/sidebarsUnversioned.js index cc95731a4..7cda202f4 100644 --- a/crowdsec-docs/sidebarsUnversioned.js +++ b/crowdsec-docs/sidebarsUnversioned.js @@ -492,7 +492,6 @@ module.exports = { "getting_started/installation/pfsense", "getting_started/installation/opnsense", "getting_started/installation/whm", - "getting_started/installation/cloudways", ], }, { diff --git a/crowdsec-docs/src/components/QuickStart.js b/crowdsec-docs/src/components/QuickStart.js index 03c8c7ab6..883d53fb4 100644 --- a/crowdsec-docs/src/components/QuickStart.js +++ b/crowdsec-docs/src/components/QuickStart.js @@ -59,11 +59,12 @@ const staticData = [ text: "WHM", link: "/u/getting_started/installation/whm", }, - { - icon: cloudwaysLogo, - text: "Cloudways", - link: "/u/getting_started/installation/cloudways", - }, + // Experimental, not linking to it + // { + // icon: cloudwaysLogo, + // text: "Cloudways", + // link: "/u/getting_started/installation/cloudways", + // }, ]; export default function QuickStart() { diff --git a/crowdsec-docs/unversioned/getting_started/installation/cloudways.mdx b/crowdsec-docs/unversioned/getting_started/installation/cloudways.mdx index 489a23c75..ad7682685 100644 --- a/crowdsec-docs/unversioned/getting_started/installation/cloudways.mdx +++ b/crowdsec-docs/unversioned/getting_started/installation/cloudways.mdx @@ -1,6 +1,6 @@ --- id: cloudways -title: Cloudways (CrowdSec + WP Remediation) +title: Cloudways (CrowdSec + WP Remediation) - 🧪 Experimental pagination_prev: getting_started/pre_requisites pagination_next: getting_started/next_steps --- @@ -9,6 +9,11 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; import CodeBlock from '@theme/CodeBlock'; +# Notice +🧪 This is an experimental way to have CrowdSec run on Cloudways, it does work well to detect attacks and remediate at the wordpress level. +However remediation at the firewall level is not possible due to the limited permissions of the user. +We hope that in the future Cloudways will have a way to handle CrowdSec Remediations at the firewall level. + # Preface Cloudways is a managed cloud hosting platform that makes it easy to host websites and applications across different cloud providers. From 92a246d9ba2a3f68a31c069b27438081ad4d9ca2 Mon Sep 17 00:00:00 2001 From: jdv Date: Tue, 7 Jan 2025 09:08:17 +0100 Subject: [PATCH 14/15] title update --- .../unversioned/getting_started/installation/cloudways.mdx | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/crowdsec-docs/unversioned/getting_started/installation/cloudways.mdx b/crowdsec-docs/unversioned/getting_started/installation/cloudways.mdx index ad7682685..545ab4928 100644 --- a/crowdsec-docs/unversioned/getting_started/installation/cloudways.mdx +++ b/crowdsec-docs/unversioned/getting_started/installation/cloudways.mdx @@ -9,12 +9,13 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; import CodeBlock from '@theme/CodeBlock'; -# Notice +# Install CrowdSec on Cloudways + 🧪 This is an experimental way to have CrowdSec run on Cloudways, it does work well to detect attacks and remediate at the wordpress level. However remediation at the firewall level is not possible due to the limited permissions of the user. We hope that in the future Cloudways will have a way to handle CrowdSec Remediations at the firewall level. -# Preface +## Preface Cloudways is a managed cloud hosting platform that makes it easy to host websites and applications across different cloud providers. It gives you SSH access, but with limited permissions. From bc49943e857b761bf2c4c162b622d7eaa614beff Mon Sep 17 00:00:00 2001 From: jdv Date: Tue, 7 Jan 2025 09:17:48 +0100 Subject: [PATCH 15/15] adding comments to NOT link --- .../unversioned/getting_started/installation/cloudways.mdx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crowdsec-docs/unversioned/getting_started/installation/cloudways.mdx b/crowdsec-docs/unversioned/getting_started/installation/cloudways.mdx index 545ab4928..a91feed9e 100644 --- a/crowdsec-docs/unversioned/getting_started/installation/cloudways.mdx +++ b/crowdsec-docs/unversioned/getting_started/installation/cloudways.mdx @@ -9,6 +9,8 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; import CodeBlock from '@theme/CodeBlock'; +{/* This page is not linked in the doc on purpose, it's experimental and only for external linking purposes (reddit...) DO NOT ADD TO MENUS */} + # Install CrowdSec on Cloudways 🧪 This is an experimental way to have CrowdSec run on Cloudways, it does work well to detect attacks and remediate at the wordpress level.