From 4740c483144033acd407455fc53b13fc3a7da079 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Fri, 10 Nov 2023 14:28:18 +0100 Subject: [PATCH 1/6] Updating paths in meta roles. --- roles/_meta/aws_client_instance/meta/main.yml | 6 +++--- roles/_meta/controller/meta/main.yml | 18 +++++++++--------- roles/_meta/deploy/meta/main.yml | 18 +++++++++--------- roles/_meta/webserver/meta/main.yml | 16 ++++++++-------- 4 files changed, 29 insertions(+), 29 deletions(-) diff --git a/roles/_meta/aws_client_instance/meta/main.yml b/roles/_meta/aws_client_instance/meta/main.yml index 9e3e7dad1..ac372d64c 100644 --- a/roles/_meta/aws_client_instance/meta/main.yml +++ b/roles/_meta/aws_client_instance/meta/main.yml @@ -1,5 +1,5 @@ --- dependencies: - - { role: apt_extra_packages, apt_extra_packages: ["cloud-init"] } - - role: aws_cloudwatch_agent - - role: aws_ssm_agent + - { role: debian/apt_extra_packages, apt_extra_packages: ["cloud-init"] } + - role: debian/aws_cloudwatch_agent + - role: debian/aws_ssm_agent diff --git a/roles/_meta/controller/meta/main.yml b/roles/_meta/controller/meta/main.yml index 8038122d7..955a23613 100644 --- a/roles/_meta/controller/meta/main.yml +++ b/roles/_meta/controller/meta/main.yml @@ -1,13 +1,13 @@ --- dependencies: - - role: user_provision - - { role: ssh_server, when: ( is_local is not defined or not is_local ) } + - role: debian/user_provision + - { role: debian/ssh_server, when: ( is_local is not defined or not is_local ) } - role: _meta/common_base - role: aws/aws_credentials - - role: ce_provision - - role: gitlab - - role: gitlab_runner - - role: sops - - role: gpg_key - - role: pam_ldap - - role: pam_linotp + - role: debian/ce_provision + - role: debian/gitlab + - role: debian/gitlab_runner + - role: debian/sops + - role: debian/gpg_key + - role: debian/pam_ldap + - role: debian/pam_linotp diff --git a/roles/_meta/deploy/meta/main.yml b/roles/_meta/deploy/meta/main.yml index 05a183528..4409e5701 100644 --- a/roles/_meta/deploy/meta/main.yml +++ b/roles/_meta/deploy/meta/main.yml @@ -1,13 +1,13 @@ --- dependencies: - - role: user_provision - - { role: ssh_server, when: ( is_local is not defined or not is_local ) } + - role: debian/user_provision + - { role: debian/ssh_server, when: ( is_local is not defined or not is_local ) } - role: _meta/common_base - - role: ce_deploy + - role: debian/ce_deploy - role: aws/aws_credentials - - role: gitlab - - role: gitlab_runner - - role: sops - - role: gpg_key - - role: pam_ldap - - role: pam_linotp + - role: debian/gitlab + - role: debian/gitlab_runner + - role: debian/sops + - role: debian/gpg_key + - role: debian/pam_ldap + - role: debian/pam_linotp diff --git a/roles/_meta/webserver/meta/main.yml b/roles/_meta/webserver/meta/main.yml index 8f4f8a94d..a66530c03 100644 --- a/roles/_meta/webserver/meta/main.yml +++ b/roles/_meta/webserver/meta/main.yml @@ -1,11 +1,11 @@ --- dependencies: - - { role: ssh_server, when: ( is_local is not defined or not is_local ) } + - { role: debian/ssh_server, when: ( is_local is not defined or not is_local ) } - role: _meta/common_base - - role: user_provision - - role: user_deploy - - role: mysql_client - - role: nodejs - - role: php-cli - - role: php-fpm - - role: nginx + - role: debian/user_provision + - role: debian/user_deploy + - role: debian/mysql_client + - role: debian/nodejs + - role: debian/php-cli + - role: debian/php-fpm + - role: debian/nginx From 41f1e5b199437e3aaebd35f6a34da10d606d9089 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Fri, 10 Nov 2023 14:37:06 +0100 Subject: [PATCH 2/6] Excluding wazuh role from linting. --- .ansible-lint | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.ansible-lint b/.ansible-lint index e49016169..7f26f195c 100644 --- a/.ansible-lint +++ b/.ansible-lint @@ -1,7 +1,6 @@ skip_list: - '204' # Lines should be no longer than 160 chars - - '301' # Commands should not changei things if nothing needs doing - - '303' # Using command rather than module + - '301' # Commands should not change things if nothing needs doing - '701' # 701 No 'galaxy_info' found - unnamed-task - risky-file-permissions @@ -9,7 +8,9 @@ skip_list: - no-handler - var-naming - no-jinja-nesting - - command-instead-of-module # dupe of '303'? + - command-instead-of-module - fqcn-builtins - template-instead-of-copy # to skip over roles/ssl/tasks/copy.yml errors, temporarily. - name[template] # it doesn't like Jinja templates being in the middle of a task name, which seems silly to me. +exclude_paths: + - roles/debian/wazuh/ # imported role uses yes/no instead of true/false so always fails From fb4cf5f1dd5b8883ccba40ee109dcdd9b06c2188 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Fri, 10 Nov 2023 14:39:46 +0100 Subject: [PATCH 3/6] Removing exclusion from CI because it's in .ansible-lint. --- .github/workflows/ce-provision-lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ce-provision-lint.yml b/.github/workflows/ce-provision-lint.yml index ab77b3ad9..8e0c9f352 100644 --- a/.github/workflows/ce-provision-lint.yml +++ b/.github/workflows/ce-provision-lint.yml @@ -28,6 +28,6 @@ jobs: - name: Run linters. if: ${{ github.event.pull_request.head.ref != 'documentation' }} run: | - ansible-lint ./roles --exclude ./roles/wazuh/defaults/ + ansible-lint ./roles yamllint ./roles cd scripts && shellcheck *.sh From 26c9408a45e1f46ecf450905912268da49165a65 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Fri, 10 Nov 2023 16:10:15 +0100 Subject: [PATCH 4/6] Adding debian readme to complete sidebar in docs. --- docs/_Sidebar.md | 48 ++++++++++++++- docs/roles/contrib/systemd_timers.md | 90 ---------------------------- docs/roles/debian.md | 2 + roles/debian/README.md | 2 + 4 files changed, 51 insertions(+), 91 deletions(-) delete mode 100644 docs/roles/contrib/systemd_timers.md create mode 100644 docs/roles/debian.md create mode 100644 roles/debian/README.md diff --git a/docs/_Sidebar.md b/docs/_Sidebar.md index 079ae0cc1..4f93d9c12 100644 --- a/docs/_Sidebar.md +++ b/docs/_Sidebar.md @@ -27,7 +27,53 @@ - [Update main route for a given VPC](/roles/aws/aws_vpc_route) - [VPC](/roles/aws/aws_vpc_subnet) - [Contributed roles](/roles/contrib) - - [ansible-systemd-timer](/roles/contrib/systemd_timers) + - [Debian Packages](/roles/debian) + - [Ansible](/roles/debian/ansible) + - [APACHE](/roles/debian/apache) + - [Extra packages](/roles/debian/apt_extra_packages) + - [APT Unattended Upgrades](/roles/debian/apt_unattended_upgrades) + - [AWS CLI](/roles/debian/aws_cli) + - [AWS Cloudwatch agent](/roles/debian/aws_cloudwatch_agent) + - [EFS client](/roles/debian/aws_efs_client) + - [AWS SSM agent](/roles/debian/aws_ssm_agent) + - [ce-deploy](/roles/debian/ce_deploy) + - [Extra packages](/roles/debian/ce_dev) + - [Automated patching](/roles/debian/ce_patcher) + - [ce-provision](/roles/debian/ce_provision) + - [ClamAV](/roles/debian/clamav) + - [Firewall Config](/roles/debian/firewall_config) + - [Frontail](/roles/debian/frontail) + - [Gitlab](/roles/debian/gitlab) + - [Gitlab Runner](/roles/debian/gitlab_runner) + - [GPG Key](/roles/debian/gpg_key) + - [HA Proxy](/roles/debian/haproxy) + - [Managed /etc/hosts](/roles/debian/hosts) + - [Jenkins](/roles/debian/jenkins) + - [Jitsi](/roles/debian/jitsi) + - [LDAP Server](/roles/debian/ldap_server) + - [LHCI](/roles/debian/lhci) + - [Mount sync](/roles/debian/mount_sync) + - [MariaDB Client](/roles/debian/mysql_client) + - [MySQL Server - Oracle Community Edition](/roles/debian/mysql_server_oracle_ce) + - [NGINX](/roles/debian/nginx) + - [NodeJS](/roles/debian/nodejs) + - [OpenVPN Config](/roles/debian/openvpn_config) + - [OSSEC](/roles/debian/ossec) + - [Packer](/roles/debian/packer) + - [PHP Composer](/roles/debian/php_composer) + - [PHP XDebug](/roles/debian/php_xdebug) + - [Postfix](/roles/debian/postfix) + - [Process Manager](/roles/debian/process_manager) + - [Python Boto](/roles/debian/python_boto) + - [Python Common](/roles/debian/python_common) + - [rkhunter](/roles/debian/rkhunter) + - [Rsyslog](/roles/debian/rsyslog) + - [solr](/roles/debian/solr) + - [SSHD](/roles/debian/ssh_server) + - [SSL](/roles/debian/ssl) + - [sudo config](/roles/debian/sudo_config) + - [varnish-config](/roles/debian/varnish_config) + - [wazuh](/roles/debian/wazuh) - [Init role](/roles/_init) - ["Meta" roles that group individual roles together.](/roles/_meta) - [AWS account](/roles/_meta/aws_account) diff --git a/docs/roles/contrib/systemd_timers.md b/docs/roles/contrib/systemd_timers.md deleted file mode 100644 index ce1e6efad..000000000 --- a/docs/roles/contrib/systemd_timers.md +++ /dev/null @@ -1,90 +0,0 @@ -# ansible-systemd-timer - -## About -This roles enables you to create systemd timers which call scripts or execute commands. - -## Usage - -Define a variable ```timers```. This variable is a dictionary. Every key is a new timer. - -### Example - -Here is an example for my 1337 Telegram Bot. The Timer "calls" a script which sends the message "It's now 13:37" in one of my Telegram chats every day at 13:37 GMT o'Clock. - -``` -timers: - 1337TelegramBot: - timer_precommand: /bin/bash -c '! /usr/bin/systemctl is-active --quiet other-service.service' - timer_command: /home/telegrambot/sendMessage.pl - timer_user: telegrambot - timer_OnCalendar: "*-*-* 13:37:00 CET" - timer_AccuracySec: 5s -``` - -That's all the magic. - -### Existing variables per timer - -| Variable | Required | Default value / Explanation | -|----------|----------|------------------------------| -| timer_precommand | no | Pre-command before command | -| timer_command | yes | Which command or script to execute | -| timer_envfile | no | Add environment file | -| timer_user | no | Under which users the timer_command is executed. Default: root | -| timer_persistent | no | Takes a boolean argument. If true, the time when the service unit was last triggered is stored on disk. When the timer is activated, the service unit is triggered immediately if it would have been triggered at least once during the time when the timer was inactive. This is useful to catch up on missed runs of the service when the machine was off. Note that this setting only has an effect on timers configured with OnCalendar=. Defaults to false. [Source](https://www.freedesktop.org/software/systemd/man/systemd.timer.html) | -| timer_workingdir | no | Set [WorkingDirectory=](https://www.freedesktop.org/software/systemd/man/systemd.exec.html#WorkingDirectory=) for the timer -| timer_OnActiveSec | no | Relative time after the timer unit was last activated | -| timer_OnBootSec | no | Relative time after the computer was booted | -| timer_OnStartupSec | no | Relative time after systemd was started | -| timer_OnUnitActiveSec | no | Relative time after the service unit was last activated | -| timer_OnUnitInactiveSec | no | Relative time after the service unit was last deactivated | -| timer_OnCalendar | no | Absolute time when to call activate the unit | -| timer_AccuracySec | no | Timer have a default accuracy of round about one minute. You can set the accuracy with this var. Default: 15s | - -You can chain every timer_On* variable. Example: - -``` -timers: - updateDNS: - timer_command: /home/dnsupdate/updateMe.pl - timer_user: dnsupdate - timer_OnStartupSec: 20s - timer_OnUnitActiveSec: 5m -``` - -The timer unit will be triggered 20 seconds after systemd was started and then every 5 minutes. - -More about timers: https://www.freedesktop.org/software/systemd/man/systemd.timer.html - -More about timespans: https://www.freedesktop.org/software/systemd/man/systemd.time.html - -### Existing variables globally, for the role -| Variable | Required | Default value / Explanation | -|----------|----------|--------------------------------------------------------------------------------------------------------------------------------------------------------| -| systemd_scope | no | Create system or user units. Default: `system`. | -| systemd_base_path | no | Where to generate the systemd unit files. Set this to e.g. `~/.config/systemd/user` when using *systemd_scope*=`user`. Default: `/etc/systemd/system`. | - -You can create user timers for non-root services in combination with `become_user: '{{ my_user }}'`. Example: -``` -systemd_base_path: ~/.config/systemd/user -systemd_scope: user -timers: - timer-one: - timer_command: ... - timer_OnCalendar: ... - timer_user: '{{ my_user }}' - ... -``` - -## Working with shell redirection - -Shell redirection does not work out of the box. You have to work around that by calling `sh` or `bash`. -This won't work: `echo hello > /var/log/hello.log` -This will work: `/usr/bin/bash -c \"echo hello > /var/log/hello.log\"` - -Tip: Always use full paths. To see where `sh` or `bash` is stored on your system you have to use `which`: - -``` -[root@pizza ~]# which bash -/usr/bin/bash -``` diff --git a/docs/roles/debian.md b/docs/roles/debian.md new file mode 100644 index 000000000..80ad608b5 --- /dev/null +++ b/docs/roles/debian.md @@ -0,0 +1,2 @@ +# Debian Packages +This is a set of roles for installing packages on Debian Linux servers. We currently support Debian versions 11 (bullseye) and 12 (bookworm). diff --git a/roles/debian/README.md b/roles/debian/README.md new file mode 100644 index 000000000..80ad608b5 --- /dev/null +++ b/roles/debian/README.md @@ -0,0 +1,2 @@ +# Debian Packages +This is a set of roles for installing packages on Debian Linux servers. We currently support Debian versions 11 (bullseye) and 12 (bookworm). From b6ebf178d9492d6d88f1d92936579f00d2377158 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Fri, 10 Nov 2023 18:29:48 +0100 Subject: [PATCH 5/6] Being more generic with Python version for venv to avoid Debian version issues. --- docs/roles/debian/ce_deploy.md | 2 +- docs/roles/debian/ce_provision.md | 2 +- roles/debian/ce_deploy/README.md | 2 +- roles/debian/ce_deploy/defaults/main.yml | 2 +- roles/debian/ce_provision/README.md | 2 +- roles/debian/ce_provision/defaults/main.yml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/roles/debian/ce_deploy.md b/docs/roles/debian/ce_deploy.md index f96ab5935..66d7402b2 100644 --- a/docs/roles/debian/ce_deploy.md +++ b/docs/roles/debian/ce_deploy.md @@ -13,7 +13,7 @@ _ce_deploy: ce_deploy: # Location of Ansible installation and components. venv_path: "/home/{{ _ce_deploy.username }}/ansible" - venv_command: /usr/bin/python3.11 -m venv + venv_command: /usr/bin/python3 -m venv # Other ce-deploy settings. new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user key_name: id_rsa.pub # existing users may have a key of a different name diff --git a/docs/roles/debian/ce_provision.md b/docs/roles/debian/ce_provision.md index f73428cbd..741ca5a6c 100644 --- a/docs/roles/debian/ce_provision.md +++ b/docs/roles/debian/ce_provision.md @@ -14,7 +14,7 @@ _ce_provision: ce_provision: # Location of Ansible installation and components. venv_path: "/home/{{ _ce_provision.username }}/ansible" - venv_command: /usr/bin/python3.11 -m venv + venv_command: /usr/bin/python3 -m venv # Other ce-provision settings. username: "{{ _ce_provision.username }}" new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user diff --git a/roles/debian/ce_deploy/README.md b/roles/debian/ce_deploy/README.md index f96ab5935..66d7402b2 100644 --- a/roles/debian/ce_deploy/README.md +++ b/roles/debian/ce_deploy/README.md @@ -13,7 +13,7 @@ _ce_deploy: ce_deploy: # Location of Ansible installation and components. venv_path: "/home/{{ _ce_deploy.username }}/ansible" - venv_command: /usr/bin/python3.11 -m venv + venv_command: /usr/bin/python3 -m venv # Other ce-deploy settings. new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user key_name: id_rsa.pub # existing users may have a key of a different name diff --git a/roles/debian/ce_deploy/defaults/main.yml b/roles/debian/ce_deploy/defaults/main.yml index 7902af13f..92fefc0ed 100644 --- a/roles/debian/ce_deploy/defaults/main.yml +++ b/roles/debian/ce_deploy/defaults/main.yml @@ -5,7 +5,7 @@ _ce_deploy: ce_deploy: # Location of Ansible installation and components. venv_path: "/home/{{ _ce_deploy.username }}/ansible" - venv_command: /usr/bin/python3.11 -m venv + venv_command: /usr/bin/python3 -m venv # Other ce-deploy settings. new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user key_name: id_rsa.pub # existing users may have a key of a different name diff --git a/roles/debian/ce_provision/README.md b/roles/debian/ce_provision/README.md index f73428cbd..741ca5a6c 100644 --- a/roles/debian/ce_provision/README.md +++ b/roles/debian/ce_provision/README.md @@ -14,7 +14,7 @@ _ce_provision: ce_provision: # Location of Ansible installation and components. venv_path: "/home/{{ _ce_provision.username }}/ansible" - venv_command: /usr/bin/python3.11 -m venv + venv_command: /usr/bin/python3 -m venv # Other ce-provision settings. username: "{{ _ce_provision.username }}" new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user diff --git a/roles/debian/ce_provision/defaults/main.yml b/roles/debian/ce_provision/defaults/main.yml index 3281cb118..f69f7c939 100644 --- a/roles/debian/ce_provision/defaults/main.yml +++ b/roles/debian/ce_provision/defaults/main.yml @@ -6,7 +6,7 @@ _ce_provision: ce_provision: # Location of Ansible installation and components. venv_path: "/home/{{ _ce_provision.username }}/ansible" - venv_command: /usr/bin/python3.11 -m venv + venv_command: /usr/bin/python3 -m venv # Other ce-provision settings. username: "{{ _ce_provision.username }}" new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user From 465b5dd8e77a75ec5da325729c78d507d6f5e584 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Fri, 10 Nov 2023 19:26:53 +0100 Subject: [PATCH 6/6] We shouldn't need to remove all the old Ansible system stuff any more. --- ce-dev/ce-dev.compose.yml | 2 +- roles/debian/ansible/tasks/main.yml | 23 ----------------------- 2 files changed, 1 insertion(+), 24 deletions(-) diff --git a/ce-dev/ce-dev.compose.yml b/ce-dev/ce-dev.compose.yml index 78a4ac0f8..227633579 100644 --- a/ce-dev/ce-dev.compose.yml +++ b/ce-dev/ce-dev.compose.yml @@ -7,7 +7,7 @@ x-ce_dev: - ce-dev/ansible/setup.yml services: controller: - image: codeenigma/ce-dev-1.x:devel + image: codeenigma/ce-dev-controller-1.x:devel x-ce_dev: {} volumes: - ../:/home/ce-dev/ce-provision diff --git a/roles/debian/ansible/tasks/main.yml b/roles/debian/ansible/tasks/main.yml index d47d0f5d9..9c7dc523d 100644 --- a/roles/debian/ansible/tasks/main.yml +++ b/roles/debian/ansible/tasks/main.yml @@ -1,27 +1,4 @@ --- -- name: Remove old PPA repository key for Ansible. - ansible.builtin.apt_key: - id: 93C4A3FD7BB9C367 - keyserver: keyserver.ubuntu.com - state: absent - -- name: Remove old repository for Ansible. - ansible.builtin.apt_repository: - repo: "deb http://ppa.launchpad.net/ansible/ansible/ubuntu trusty main" - state: absent - -# Linters need a newer version of PyYAML than the one that ships with Debian. -- name: Remove older PyYAML package. - ansible.builtin.apt: - pkg: python3-yaml - state: absent - when: ce_ansible.linters.enabled - -- name: Remove system Ansible. - ansible.builtin.apt: - pkg: ansible - state: absent - - name: Set up Python packages. ansible.builtin.include_role: name: debian/python_common