From 90aeb2c5638643dad10808e36ec9e4b691f38591 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Wed, 22 Nov 2023 18:16:51 +0100 Subject: [PATCH 01/33] Improving AWS subnet docs. --- docs/roles/aws/aws_ec2_autoscale_cluster.md | 2 ++ docs/roles/aws/aws_rds.md | 1 + docs/roles/aws/aws_vpc_subnet.md | 1 + roles/aws/aws_ec2_autoscale_cluster/README.md | 2 ++ roles/aws/aws_ec2_autoscale_cluster/defaults/main.yml | 2 ++ roles/aws/aws_rds/README.md | 1 + roles/aws/aws_rds/defaults/main.yml | 1 + roles/aws/aws_vpc_subnet/README.md | 1 + roles/aws/aws_vpc_subnet/defaults/main.yml | 1 + 9 files changed, 12 insertions(+) diff --git a/docs/roles/aws/aws_ec2_autoscale_cluster.md b/docs/roles/aws/aws_ec2_autoscale_cluster.md index 5e6909609..49f419391 100644 --- a/docs/roles/aws/aws_ec2_autoscale_cluster.md +++ b/docs/roles/aws/aws_ec2_autoscale_cluster.md @@ -36,6 +36,8 @@ aws_ec2_autoscale_cluster: vpc_id: vpc-XXXX # One of vpc_id or vpc_name is mandatory. # vpc_name: example-vpc subnets: + # If rds.rds.true is set and you are creating an RDS instance then you must use at least two AZs, this is hard requirement. + # This means your region and environment must also have a minimum of two subnets - see `aws_vpc_subnet` # - az: a # cidr: "10.0.3.0/26" - az: b diff --git a/docs/roles/aws/aws_rds.md b/docs/roles/aws/aws_rds.md index e89cd6e74..1e5287a15 100644 --- a/docs/roles/aws/aws_rds.md +++ b/docs/roles/aws/aws_rds.md @@ -19,6 +19,7 @@ aws_rds: aws_profile: "{{ _aws_profile }}" region: "{{ _aws_region }}" multi_az: true + # You must provide at least two subnets in two different AZs, even for single AZ deployments. subnets: - subnet-aaaaaaaa - subnet-bbbbbbbb diff --git a/docs/roles/aws/aws_vpc_subnet.md b/docs/roles/aws/aws_vpc_subnet.md index 5d97d303a..1b4ef012b 100644 --- a/docs/roles/aws/aws_vpc_subnet.md +++ b/docs/roles/aws/aws_vpc_subnet.md @@ -11,6 +11,7 @@ aws_vpc_subnet: # vpc_name: example-vpc aws_profile: "{{ _aws_profile }}" region: "{{ _aws_region }}" + # If you intend to have RDS instances you must provide at least two subnets. subnets: - cidr_block: "10.0.0.0/24" # ipv6_cidr_block: "1" # This will create something like xxxx:xxxx:xxxx:xxyy::/64 where yy is created using the ansible.utils.ipsubnet filter automatically - DO NOT DEFINE IF IPV6 IS NOT REQUIRED diff --git a/roles/aws/aws_ec2_autoscale_cluster/README.md b/roles/aws/aws_ec2_autoscale_cluster/README.md index 5e6909609..49f419391 100644 --- a/roles/aws/aws_ec2_autoscale_cluster/README.md +++ b/roles/aws/aws_ec2_autoscale_cluster/README.md @@ -36,6 +36,8 @@ aws_ec2_autoscale_cluster: vpc_id: vpc-XXXX # One of vpc_id or vpc_name is mandatory. # vpc_name: example-vpc subnets: + # If rds.rds.true is set and you are creating an RDS instance then you must use at least two AZs, this is hard requirement. + # This means your region and environment must also have a minimum of two subnets - see `aws_vpc_subnet` # - az: a # cidr: "10.0.3.0/26" - az: b diff --git a/roles/aws/aws_ec2_autoscale_cluster/defaults/main.yml b/roles/aws/aws_ec2_autoscale_cluster/defaults/main.yml index 5bdd39aef..9af962f5c 100644 --- a/roles/aws/aws_ec2_autoscale_cluster/defaults/main.yml +++ b/roles/aws/aws_ec2_autoscale_cluster/defaults/main.yml @@ -17,6 +17,8 @@ aws_ec2_autoscale_cluster: vpc_id: vpc-XXXX # One of vpc_id or vpc_name is mandatory. # vpc_name: example-vpc subnets: + # If rds.rds.true is set and you are creating an RDS instance then you must use at least two AZs, this is hard requirement. + # This means your region and environment must also have a minimum of two subnets - see `aws_vpc_subnet` # - az: a # cidr: "10.0.3.0/26" - az: b diff --git a/roles/aws/aws_rds/README.md b/roles/aws/aws_rds/README.md index e89cd6e74..1e5287a15 100644 --- a/roles/aws/aws_rds/README.md +++ b/roles/aws/aws_rds/README.md @@ -19,6 +19,7 @@ aws_rds: aws_profile: "{{ _aws_profile }}" region: "{{ _aws_region }}" multi_az: true + # You must provide at least two subnets in two different AZs, even for single AZ deployments. subnets: - subnet-aaaaaaaa - subnet-bbbbbbbb diff --git a/roles/aws/aws_rds/defaults/main.yml b/roles/aws/aws_rds/defaults/main.yml index 96b4d4914..faa3ee5bb 100644 --- a/roles/aws/aws_rds/defaults/main.yml +++ b/roles/aws/aws_rds/defaults/main.yml @@ -2,6 +2,7 @@ aws_rds: aws_profile: "{{ _aws_profile }}" region: "{{ _aws_region }}" multi_az: true + # You must provide at least two subnets in two different AZs, even for single AZ deployments. subnets: - subnet-aaaaaaaa - subnet-bbbbbbbb diff --git a/roles/aws/aws_vpc_subnet/README.md b/roles/aws/aws_vpc_subnet/README.md index 5d97d303a..1b4ef012b 100644 --- a/roles/aws/aws_vpc_subnet/README.md +++ b/roles/aws/aws_vpc_subnet/README.md @@ -11,6 +11,7 @@ aws_vpc_subnet: # vpc_name: example-vpc aws_profile: "{{ _aws_profile }}" region: "{{ _aws_region }}" + # If you intend to have RDS instances you must provide at least two subnets. subnets: - cidr_block: "10.0.0.0/24" # ipv6_cidr_block: "1" # This will create something like xxxx:xxxx:xxxx:xxyy::/64 where yy is created using the ansible.utils.ipsubnet filter automatically - DO NOT DEFINE IF IPV6 IS NOT REQUIRED diff --git a/roles/aws/aws_vpc_subnet/defaults/main.yml b/roles/aws/aws_vpc_subnet/defaults/main.yml index 1e29a7d01..e72b3ae64 100644 --- a/roles/aws/aws_vpc_subnet/defaults/main.yml +++ b/roles/aws/aws_vpc_subnet/defaults/main.yml @@ -3,6 +3,7 @@ aws_vpc_subnet: # vpc_name: example-vpc aws_profile: "{{ _aws_profile }}" region: "{{ _aws_region }}" + # If you intend to have RDS instances you must provide at least two subnets. subnets: - cidr_block: "10.0.0.0/24" # ipv6_cidr_block: "1" # This will create something like xxxx:xxxx:xxxx:xxyy::/64 where yy is created using the ansible.utils.ipsubnet filter automatically - DO NOT DEFINE IF IPV6 IS NOT REQUIRED From 7c31b40ef78d23c459b7c05ed01463edfe756255 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Tue, 12 Dec 2023 18:34:14 +0100 Subject: [PATCH 02/33] Error in timers structure in the SSL role. --- roles/debian/nginx/defaults/main.yml | 1 + roles/debian/ssl/tasks/letsencrypt.yml | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/debian/nginx/defaults/main.yml b/roles/debian/nginx/defaults/main.yml index 61f2be571..64dbc76fa 100644 --- a/roles/debian/nginx/defaults/main.yml +++ b/roles/debian/nginx/defaults/main.yml @@ -57,6 +57,7 @@ nginx: # reload_command: restart # reload: # - nginx + # on_calendar: "Mon *-*-* 04:00:00" ratelimitingcrawlers: true is_default: true basic_auth: diff --git a/roles/debian/ssl/tasks/letsencrypt.yml b/roles/debian/ssl/tasks/letsencrypt.yml index 616040ae1..65ff333b4 100644 --- a/roles/debian/ssl/tasks/letsencrypt.yml +++ b/roles/debian/ssl/tasks/letsencrypt.yml @@ -87,6 +87,5 @@ ansible.builtin.include_role: name: contrib/systemd_timers vars: - timers: - certbot_renewal: "{{ _certbot_renewal_timer }}" + timers: "{{ _certbot_renewal_timer }}" when: ssl.autorenew From b75fe03127d42033ed1b4927f2f89ac78d4a78d4 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Thu, 21 Dec 2023 16:00:54 +0100 Subject: [PATCH 03/33] Removing obsolete backports requirements. --- roles/debian/firewall_config/tasks/main.yml | 10 ------ roles/debian/ldap_server/tasks/main.yml | 11 ------- roles/debian/lhci/tasks/main.yml | 34 --------------------- 3 files changed, 55 deletions(-) diff --git a/roles/debian/firewall_config/tasks/main.yml b/roles/debian/firewall_config/tasks/main.yml index 66941b51a..cf770ab4e 100644 --- a/roles/debian/firewall_config/tasks/main.yml +++ b/roles/debian/firewall_config/tasks/main.yml @@ -1,14 +1,4 @@ --- -# @TODO remove when https://redmine.codeenigma.net/issues/58246 can be done -- name: Install iptables with backports. # we have to do this in ce-dev or the contrib role will fail - ansible.builtin.apt: - pkg: ["iptables"] - state: present - default_release: buster-backports - when: - - is_local is defined - - is_local - - name: Shift general firewall settings to expected variables. ansible.builtin.set_fact: firewall_state: "{{ firewall_config.firewall_state }}" diff --git a/roles/debian/ldap_server/tasks/main.yml b/roles/debian/ldap_server/tasks/main.yml index 52fb1ccf4..9a773da58 100644 --- a/roles/debian/ldap_server/tasks/main.yml +++ b/roles/debian/ldap_server/tasks/main.yml @@ -14,21 +14,10 @@ purge: true when: ldap_server.slapd.purge -# @TODO remove when https://redmine.codeenigma.net/issues/58246 can be done -- name: Ensure LDAP and dependencies are installed from backports. - ansible.builtin.apt: - pkg: ["slapd", "ldapscripts", "libldap2-dev"] # python-ldap needs libldap2-dev - state: present - default_release: buster-backports - when: - - is_local is defined - - is_local - - name: Ensure LDAP and python-ldap and dependencies are installed. ansible.builtin.apt: pkg: ["slapd", "ldapscripts", "libldap2-dev"] state: present - when: is_local is not defined - name: Ensure additional dependencies for python-ldap are installed. ansible.builtin.apt: diff --git a/roles/debian/lhci/tasks/main.yml b/roles/debian/lhci/tasks/main.yml index c92447d4b..56c1fd9c9 100644 --- a/roles/debian/lhci/tasks/main.yml +++ b/roles/debian/lhci/tasks/main.yml @@ -10,26 +10,6 @@ state: present filename: google-chrome -# @TODO remove when https://redmine.codeenigma.net/issues/58246 can be done -- name: Install Google Chrome "headful" mode dependencies from backports. - ansible.builtin.apt: - name: - - xorg - - xvfb - - gtk2-engines-pixbuf - - dbus-x11 - - xfonts-base - - xfonts-100dpi - - xfonts-75dpi - - xfonts-cyrillic - - xfonts-scalable - default_release: buster-backports - state: present - when: - - is_local - - ansible_distribution == "Debian" - - ansible_distribution_major_version == "10" - - name: Install Google Chrome "headful" mode dependencies. ansible.builtin.apt: name: @@ -43,19 +23,6 @@ - xfonts-cyrillic - xfonts-scalable state: present - when: is_local is not defined - -# @TODO remove when https://redmine.codeenigma.net/issues/58246 can be done -- name: Install VNC support from backports. - ansible.builtin.apt: - name: x11vnc - default_release: buster-backports - state: present - when: - - lhci.enable_vnc - - is_local - - ansible_distribution == "Debian" - - ansible_distribution_major_version == "10" - name: Install VNC support. ansible.builtin.apt: @@ -63,7 +30,6 @@ state: present when: - lhci.enable_vnc - - is_local is not defined - name: Configure Xvfb to start on boot. ansible.builtin.shell: | From 5b27a8c0f0d7a9f8f1fd8437c96fafe33f98a692 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Thu, 21 Dec 2023 16:46:16 +0100 Subject: [PATCH 04/33] Allow the billing role to access Sustainability information. --- roles/aws/aws_iam_saml/templates/access_billing_policy.j2 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/roles/aws/aws_iam_saml/templates/access_billing_policy.j2 b/roles/aws/aws_iam_saml/templates/access_billing_policy.j2 index cf20372f8..714056642 100644 --- a/roles/aws/aws_iam_saml/templates/access_billing_policy.j2 +++ b/roles/aws/aws_iam_saml/templates/access_billing_policy.j2 @@ -9,7 +9,8 @@ "freetier:*", "ce:*", "cur:*", - "tax:*" + "tax:*", + "sustainability:*" ], "Effect": "Allow", "Resource": "*" From 9ca4148dddc42a0d5e102562902f2d017500b996 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Thu, 21 Dec 2023 18:27:48 +0100 Subject: [PATCH 05/33] Missing comma in IAM billing policy. --- roles/aws/aws_iam_saml/templates/access_billing_policy.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/aws/aws_iam_saml/templates/access_billing_policy.j2 b/roles/aws/aws_iam_saml/templates/access_billing_policy.j2 index 714056642..b26f9e7c7 100644 --- a/roles/aws/aws_iam_saml/templates/access_billing_policy.j2 +++ b/roles/aws/aws_iam_saml/templates/access_billing_policy.j2 @@ -28,7 +28,7 @@ "tax:BatchPutTaxRegistration", "tax:DeleteTaxRegistration", "tax:PutTaxInheritance" - ] + ], "Effect": "Deny", "Resource": "*" } From c8199bef92edb321c8df6e7d11a361c98227941e Mon Sep 17 00:00:00 2001 From: gregharvey Date: Tue, 9 Jan 2024 19:30:01 +0100 Subject: [PATCH 06/33] Removing broken GitLab Runner code. --- roles/debian/gitlab_runner/tasks/main.yml | 25 +++++++++++-------- .../gitlab_runner/templates/fargate.toml.j2 | 2 +- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/roles/debian/gitlab_runner/tasks/main.yml b/roles/debian/gitlab_runner/tasks/main.yml index 2601d8e4f..37a30227a 100644 --- a/roles/debian/gitlab_runner/tasks/main.yml +++ b/roles/debian/gitlab_runner/tasks/main.yml @@ -35,18 +35,21 @@ # gitlab-runner register --url https://gitlab.com/ --registration-token TOKEN_HERE --name fargate-test-runner --run-untagged --executor custom -n # This will automatically update /etc/gitlab-runner/config.toml +# @TODO this needs rethinking, we cannot delegate include_role! +# Replace security_group in template with _aws_security_group_list[0] when resolved + # Populates the _aws_security_group_list variable used in fargate.toml.j2 -- name: Generate security group information. - ansible.builtin.include_role: - name: aws/aws_security_groups - vars: - profile: "{{ aws_elasticache.aws_profile }}" - region: "{{ aws_elasticache.region }}" - group_names: - - "{{ gitlab_runner.fargate.security_group }}" - return_type: ids - when: gitlab_runner.install_fargate - delegate_to: localhost +#- name: Generate security group information. +# ansible.builtin.include_role: +# name: aws/aws_security_groups +# vars: +# profile: "{{ gitlab_runner.fargate.region }}" +# region: "{{ gitlab_runner.fargate.region }}" +# group_names: +# - "{{ gitlab_runner.fargate.security_group }}" +# return_type: ids +# when: gitlab_runner.install_fargate +# delegate_to: localhost - name: Create the Fargate driver directory if it does not exist. ansible.builtin.file: diff --git a/roles/debian/gitlab_runner/templates/fargate.toml.j2 b/roles/debian/gitlab_runner/templates/fargate.toml.j2 index 6af78fc5a..c32c3633a 100644 --- a/roles/debian/gitlab_runner/templates/fargate.toml.j2 +++ b/roles/debian/gitlab_runner/templates/fargate.toml.j2 @@ -5,7 +5,7 @@ LogFormat = "text" Cluster = "{{ gitlab_runner.fargate.cluster }}" Region = "{{ gitlab_runner.fargate.region }}" Subnet = "{{ gitlab_runner.fargate.subnet }}" - SecurityGroup = "{{ _aws_security_group_list[0] }}" + SecurityGroup = "{{ gitlab_runner.fargate.security_group }}" TaskDefinition = "{{ gitlab_runner.fargate.task_definition }}" EnablePublicIP = {{ gitlab_runner.fargate.public_ip }} PlatformVersion = "{{ gitlab_runner.fargate.version }}" From 0ba41388c95088c6befa309343268d6c21459f25 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Wed, 10 Jan 2024 10:58:45 +0100 Subject: [PATCH 07/33] Fixed the include_role task in gitlab_runner. --- docs/roles/debian/gitlab_runner.md | 1 + roles/debian/gitlab_runner/README.md | 1 + roles/debian/gitlab_runner/defaults/main.yml | 1 + roles/debian/gitlab_runner/tasks/main.yml | 26 +++++++++---------- .../gitlab_runner/templates/fargate.toml.j2 | 2 +- 5 files changed, 16 insertions(+), 15 deletions(-) diff --git a/docs/roles/debian/gitlab_runner.md b/docs/roles/debian/gitlab_runner.md index 52554ced5..10829036f 100644 --- a/docs/roles/debian/gitlab_runner.md +++ b/docs/roles/debian/gitlab_runner.md @@ -48,6 +48,7 @@ gitlab_runner: # see https://gitlab.com/gitlab-org/ci-cd/custom-executor-drivers/fargate/-/tree/master/docs fargate: cluster: "my-cluster" # ECS cluster name + profile: "{{ _aws_profile }}" region: "eu-west-1" # AWS region name subnet: "subnet-abcdef123456" # subnet ID security_group: "my-security-group" # SG name diff --git a/roles/debian/gitlab_runner/README.md b/roles/debian/gitlab_runner/README.md index 52554ced5..10829036f 100644 --- a/roles/debian/gitlab_runner/README.md +++ b/roles/debian/gitlab_runner/README.md @@ -48,6 +48,7 @@ gitlab_runner: # see https://gitlab.com/gitlab-org/ci-cd/custom-executor-drivers/fargate/-/tree/master/docs fargate: cluster: "my-cluster" # ECS cluster name + profile: "{{ _aws_profile }}" region: "eu-west-1" # AWS region name subnet: "subnet-abcdef123456" # subnet ID security_group: "my-security-group" # SG name diff --git a/roles/debian/gitlab_runner/defaults/main.yml b/roles/debian/gitlab_runner/defaults/main.yml index 845371447..ba1c1d48e 100644 --- a/roles/debian/gitlab_runner/defaults/main.yml +++ b/roles/debian/gitlab_runner/defaults/main.yml @@ -36,6 +36,7 @@ gitlab_runner: # see https://gitlab.com/gitlab-org/ci-cd/custom-executor-drivers/fargate/-/tree/master/docs fargate: cluster: "my-cluster" # ECS cluster name + profile: "{{ _aws_profile }}" region: "eu-west-1" # AWS region name subnet: "subnet-abcdef123456" # subnet ID security_group: "my-security-group" # SG name diff --git a/roles/debian/gitlab_runner/tasks/main.yml b/roles/debian/gitlab_runner/tasks/main.yml index 37a30227a..685b0bfc2 100644 --- a/roles/debian/gitlab_runner/tasks/main.yml +++ b/roles/debian/gitlab_runner/tasks/main.yml @@ -35,21 +35,19 @@ # gitlab-runner register --url https://gitlab.com/ --registration-token TOKEN_HERE --name fargate-test-runner --run-untagged --executor custom -n # This will automatically update /etc/gitlab-runner/config.toml -# @TODO this needs rethinking, we cannot delegate include_role! -# Replace security_group in template with _aws_security_group_list[0] when resolved - # Populates the _aws_security_group_list variable used in fargate.toml.j2 -#- name: Generate security group information. -# ansible.builtin.include_role: -# name: aws/aws_security_groups -# vars: -# profile: "{{ gitlab_runner.fargate.region }}" -# region: "{{ gitlab_runner.fargate.region }}" -# group_names: -# - "{{ gitlab_runner.fargate.security_group }}" -# return_type: ids -# when: gitlab_runner.install_fargate -# delegate_to: localhost +- name: Generate security group information. + ansible.builtin.include_role: + name: aws/aws_security_groups + apply: + delegate_to: localhost # this is how you delegate the include_role module + vars: + profile: "{{ gitlab_runner.fargate.profile }}" + region: "{{ gitlab_runner.fargate.region }}" + group_names: + - "{{ gitlab_runner.fargate.security_group }}" + return_type: ids + when: gitlab_runner.install_fargate - name: Create the Fargate driver directory if it does not exist. ansible.builtin.file: diff --git a/roles/debian/gitlab_runner/templates/fargate.toml.j2 b/roles/debian/gitlab_runner/templates/fargate.toml.j2 index c32c3633a..6af78fc5a 100644 --- a/roles/debian/gitlab_runner/templates/fargate.toml.j2 +++ b/roles/debian/gitlab_runner/templates/fargate.toml.j2 @@ -5,7 +5,7 @@ LogFormat = "text" Cluster = "{{ gitlab_runner.fargate.cluster }}" Region = "{{ gitlab_runner.fargate.region }}" Subnet = "{{ gitlab_runner.fargate.subnet }}" - SecurityGroup = "{{ gitlab_runner.fargate.security_group }}" + SecurityGroup = "{{ _aws_security_group_list[0] }}" TaskDefinition = "{{ gitlab_runner.fargate.task_definition }}" EnablePublicIP = {{ gitlab_runner.fargate.public_ip }} PlatformVersion = "{{ gitlab_runner.fargate.version }}" From aacf70beb8415c637571707c37c6dc5d59704ea0 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Wed, 10 Jan 2024 11:00:16 +0100 Subject: [PATCH 08/33] Suppressing a failure if there is no system pip to call. --- roles/debian/ansible/tasks/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/debian/ansible/tasks/main.yml b/roles/debian/ansible/tasks/main.yml index 722690a64..d5f414624 100644 --- a/roles/debian/ansible/tasks/main.yml +++ b/roles/debian/ansible/tasks/main.yml @@ -10,6 +10,7 @@ state: absent executable: pip3 when: ansible_distribution_major_version | int < 12 + failed_when: false # don't stop the build if there's no system pip - name: Set up Python packages. ansible.builtin.include_role: From c634ef5e61ced0403f781263549e731833c4121d Mon Sep 17 00:00:00 2001 From: gregharvey Date: Wed, 10 Jan 2024 11:19:53 +0100 Subject: [PATCH 09/33] Logic error in Ansible installer username, needs to be set from calling role. --- docs/roles/debian/ansible.md | 3 ++- docs/roles/debian/ce_deploy.md | 1 + docs/roles/debian/ce_provision.md | 1 + roles/debian/ansible/README.md | 3 ++- roles/debian/ansible/defaults/main.yml | 3 ++- roles/debian/ansible/tasks/main.yml | 6 +++--- roles/debian/ce_deploy/README.md | 1 + roles/debian/ce_deploy/defaults/main.yml | 1 + roles/debian/ce_deploy/tasks/main.yml | 1 + roles/debian/ce_provision/README.md | 1 + roles/debian/ce_provision/defaults/main.yml | 1 + roles/debian/ce_provision/tasks/main.yml | 1 + 12 files changed, 17 insertions(+), 6 deletions(-) diff --git a/docs/roles/debian/ansible.md b/docs/roles/debian/ansible.md index 7e0015fd9..e87cedd66 100644 --- a/docs/roles/debian/ansible.md +++ b/docs/roles/debian/ansible.md @@ -11,9 +11,10 @@ Note, it is vitally important that Ansible is *not* installed via `apt` or `pip` ```yaml --- ce_ansible: - # These are usually set within another role using _venv_path and _venv_command but can be overridden. + # These are usually set within another role using _venv_path, _venv_command and _ansible_user but can be overridden. #venv_path: "/home/{{ ce_provision.username }}/ansible" #venv_command: /usr/bin/python3.11 -m venv + #ansible_user: deploy # user to become when creating venv upgrade: enabled: true # create systemd timer to auto-upgrade Ansible command: "{{ _venv_path }}/bin/python3 -m pip install --upgrade ansible" # if you set venv_path above then set it here too diff --git a/docs/roles/debian/ce_deploy.md b/docs/roles/debian/ce_deploy.md index 66d7402b2..d829d089f 100644 --- a/docs/roles/debian/ce_deploy.md +++ b/docs/roles/debian/ce_deploy.md @@ -14,6 +14,7 @@ ce_deploy: # Location of Ansible installation and components. venv_path: "/home/{{ _ce_deploy.username }}/ansible" venv_command: /usr/bin/python3 -m venv + ansible_user: "{{ _ce_deploy.username }}" # Other ce-deploy settings. new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user key_name: id_rsa.pub # existing users may have a key of a different name diff --git a/docs/roles/debian/ce_provision.md b/docs/roles/debian/ce_provision.md index 741ca5a6c..001d8fad6 100644 --- a/docs/roles/debian/ce_provision.md +++ b/docs/roles/debian/ce_provision.md @@ -15,6 +15,7 @@ ce_provision: # Location of Ansible installation and components. venv_path: "/home/{{ _ce_provision.username }}/ansible" venv_command: /usr/bin/python3 -m venv + ansible_user: "{{ _ce_provision.username }}" # Other ce-provision settings. username: "{{ _ce_provision.username }}" new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user diff --git a/roles/debian/ansible/README.md b/roles/debian/ansible/README.md index 7e0015fd9..e87cedd66 100644 --- a/roles/debian/ansible/README.md +++ b/roles/debian/ansible/README.md @@ -11,9 +11,10 @@ Note, it is vitally important that Ansible is *not* installed via `apt` or `pip` ```yaml --- ce_ansible: - # These are usually set within another role using _venv_path and _venv_command but can be overridden. + # These are usually set within another role using _venv_path, _venv_command and _ansible_user but can be overridden. #venv_path: "/home/{{ ce_provision.username }}/ansible" #venv_command: /usr/bin/python3.11 -m venv + #ansible_user: deploy # user to become when creating venv upgrade: enabled: true # create systemd timer to auto-upgrade Ansible command: "{{ _venv_path }}/bin/python3 -m pip install --upgrade ansible" # if you set venv_path above then set it here too diff --git a/roles/debian/ansible/defaults/main.yml b/roles/debian/ansible/defaults/main.yml index 746251010..2c6108938 100644 --- a/roles/debian/ansible/defaults/main.yml +++ b/roles/debian/ansible/defaults/main.yml @@ -1,8 +1,9 @@ --- ce_ansible: - # These are usually set within another role using _venv_path and _venv_command but can be overridden. + # These are usually set within another role using _venv_path, _venv_command and _ansible_user but can be overridden. #venv_path: "/home/{{ ce_provision.username }}/ansible" #venv_command: /usr/bin/python3.11 -m venv + #ansible_user: deploy # user to become when creating venv upgrade: enabled: true # create systemd timer to auto-upgrade Ansible command: "{{ _venv_path }}/bin/python3 -m pip install --upgrade ansible" # if you set venv_path above then set it here too diff --git a/roles/debian/ansible/tasks/main.yml b/roles/debian/ansible/tasks/main.yml index d5f414624..ead087450 100644 --- a/roles/debian/ansible/tasks/main.yml +++ b/roles/debian/ansible/tasks/main.yml @@ -24,7 +24,7 @@ virtualenv: "{{ ce_ansible.venv_path | default(_venv_path) }}" virtualenv_command: "{{ ce_ansible.venv_command | default(_venv_command) }}" become: true - become_user: "{{ ce_provision.username }}" + become_user: "{{ ce_ansible.ansible_user | default(_ansible_user) }}" - name: Install Ansible. ansible.builtin.pip: @@ -33,7 +33,7 @@ virtualenv: "{{ ce_ansible.venv_path | default(_venv_path) }}" virtualenv_command: "{{ ce_ansible.venv_command | default(_venv_command) }}" become: true - become_user: "{{ ce_provision.username }}" + become_user: "{{ ce_ansible.ansible_user | default(_ansible_user) }}" - name: Install linters. ansible.builtin.pip: @@ -44,7 +44,7 @@ virtualenv_command: "{{ ce_ansible.venv_command | default(_venv_command) }}" when: ce_ansible.linters.enabled become: true - become_user: "{{ ce_provision.username }}" + become_user: "{{ ce_ansible.ansible_user | default(_ansible_user) }}" - name: Add the venv to $PATH using profile.d. ansible.builtin.copy: diff --git a/roles/debian/ce_deploy/README.md b/roles/debian/ce_deploy/README.md index 66d7402b2..d829d089f 100644 --- a/roles/debian/ce_deploy/README.md +++ b/roles/debian/ce_deploy/README.md @@ -14,6 +14,7 @@ ce_deploy: # Location of Ansible installation and components. venv_path: "/home/{{ _ce_deploy.username }}/ansible" venv_command: /usr/bin/python3 -m venv + ansible_user: "{{ _ce_deploy.username }}" # Other ce-deploy settings. new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user key_name: id_rsa.pub # existing users may have a key of a different name diff --git a/roles/debian/ce_deploy/defaults/main.yml b/roles/debian/ce_deploy/defaults/main.yml index 92fefc0ed..276780b3b 100644 --- a/roles/debian/ce_deploy/defaults/main.yml +++ b/roles/debian/ce_deploy/defaults/main.yml @@ -6,6 +6,7 @@ ce_deploy: # Location of Ansible installation and components. venv_path: "/home/{{ _ce_deploy.username }}/ansible" venv_command: /usr/bin/python3 -m venv + ansible_user: "{{ _ce_deploy.username }}" # Other ce-deploy settings. new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user key_name: id_rsa.pub # existing users may have a key of a different name diff --git a/roles/debian/ce_deploy/tasks/main.yml b/roles/debian/ce_deploy/tasks/main.yml index 723ef9f57..f047afacc 100644 --- a/roles/debian/ce_deploy/tasks/main.yml +++ b/roles/debian/ce_deploy/tasks/main.yml @@ -132,6 +132,7 @@ ansible.builtin.set_fact: _venv_path: "{{ ce_deploy.venv_path }}" _venv_command: "{{ ce_deploy.venv_command }}" + _ansible_user: "{{ ce_deploy.ansible_user }}" - name: Install Ansible. ansible.builtin.include_role: diff --git a/roles/debian/ce_provision/README.md b/roles/debian/ce_provision/README.md index 741ca5a6c..001d8fad6 100644 --- a/roles/debian/ce_provision/README.md +++ b/roles/debian/ce_provision/README.md @@ -15,6 +15,7 @@ ce_provision: # Location of Ansible installation and components. venv_path: "/home/{{ _ce_provision.username }}/ansible" venv_command: /usr/bin/python3 -m venv + ansible_user: "{{ _ce_provision.username }}" # Other ce-provision settings. username: "{{ _ce_provision.username }}" new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user diff --git a/roles/debian/ce_provision/defaults/main.yml b/roles/debian/ce_provision/defaults/main.yml index f69f7c939..35da59995 100644 --- a/roles/debian/ce_provision/defaults/main.yml +++ b/roles/debian/ce_provision/defaults/main.yml @@ -7,6 +7,7 @@ ce_provision: # Location of Ansible installation and components. venv_path: "/home/{{ _ce_provision.username }}/ansible" venv_command: /usr/bin/python3 -m venv + ansible_user: "{{ _ce_provision.username }}" # Other ce-provision settings. username: "{{ _ce_provision.username }}" new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user diff --git a/roles/debian/ce_provision/tasks/main.yml b/roles/debian/ce_provision/tasks/main.yml index 36b38bc8b..62662a1f5 100644 --- a/roles/debian/ce_provision/tasks/main.yml +++ b/roles/debian/ce_provision/tasks/main.yml @@ -140,6 +140,7 @@ ansible.builtin.set_fact: _venv_path: "{{ ce_provision.venv_path }}" _venv_command: "{{ ce_provision.venv_command }}" + _ansible_user: "{{ ce_provision.ansible_user }}" - name: Install Ansible. ansible.builtin.include_role: From 267365988cfdb39561781854e2243f7ad485e843 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Wed, 10 Jan 2024 12:40:14 +0100 Subject: [PATCH 10/33] ansible_user is a reserved variable, seems to be causing issues. --- docs/roles/debian/ansible.md | 4 ++-- docs/roles/debian/ce_deploy.md | 2 +- docs/roles/debian/ce_provision.md | 2 +- roles/debian/ansible/README.md | 4 ++-- roles/debian/ansible/defaults/main.yml | 4 ++-- roles/debian/ansible/tasks/main.yml | 6 +++--- roles/debian/ce_deploy/README.md | 2 +- roles/debian/ce_deploy/defaults/main.yml | 2 +- roles/debian/ce_deploy/tasks/main.yml | 2 +- roles/debian/ce_provision/README.md | 2 +- roles/debian/ce_provision/defaults/main.yml | 2 +- roles/debian/ce_provision/tasks/main.yml | 2 +- 12 files changed, 17 insertions(+), 17 deletions(-) diff --git a/docs/roles/debian/ansible.md b/docs/roles/debian/ansible.md index e87cedd66..faad26b84 100644 --- a/docs/roles/debian/ansible.md +++ b/docs/roles/debian/ansible.md @@ -11,10 +11,10 @@ Note, it is vitally important that Ansible is *not* installed via `apt` or `pip` ```yaml --- ce_ansible: - # These are usually set within another role using _venv_path, _venv_command and _ansible_user but can be overridden. + # These are usually set within another role using _venv_path, _venv_command and _ansible_install_username but can be overridden. #venv_path: "/home/{{ ce_provision.username }}/ansible" #venv_command: /usr/bin/python3.11 -m venv - #ansible_user: deploy # user to become when creating venv + #ansible_install_username: deploy # user to become when creating venv upgrade: enabled: true # create systemd timer to auto-upgrade Ansible command: "{{ _venv_path }}/bin/python3 -m pip install --upgrade ansible" # if you set venv_path above then set it here too diff --git a/docs/roles/debian/ce_deploy.md b/docs/roles/debian/ce_deploy.md index d829d089f..b9f3e983d 100644 --- a/docs/roles/debian/ce_deploy.md +++ b/docs/roles/debian/ce_deploy.md @@ -14,7 +14,7 @@ ce_deploy: # Location of Ansible installation and components. venv_path: "/home/{{ _ce_deploy.username }}/ansible" venv_command: /usr/bin/python3 -m venv - ansible_user: "{{ _ce_deploy.username }}" + ansible_install_username: "{{ _ce_deploy.username }}" # Other ce-deploy settings. new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user key_name: id_rsa.pub # existing users may have a key of a different name diff --git a/docs/roles/debian/ce_provision.md b/docs/roles/debian/ce_provision.md index 001d8fad6..305ec1894 100644 --- a/docs/roles/debian/ce_provision.md +++ b/docs/roles/debian/ce_provision.md @@ -15,7 +15,7 @@ ce_provision: # Location of Ansible installation and components. venv_path: "/home/{{ _ce_provision.username }}/ansible" venv_command: /usr/bin/python3 -m venv - ansible_user: "{{ _ce_provision.username }}" + ansible_install_username: "{{ _ce_provision.username }}" # Other ce-provision settings. username: "{{ _ce_provision.username }}" new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user diff --git a/roles/debian/ansible/README.md b/roles/debian/ansible/README.md index e87cedd66..faad26b84 100644 --- a/roles/debian/ansible/README.md +++ b/roles/debian/ansible/README.md @@ -11,10 +11,10 @@ Note, it is vitally important that Ansible is *not* installed via `apt` or `pip` ```yaml --- ce_ansible: - # These are usually set within another role using _venv_path, _venv_command and _ansible_user but can be overridden. + # These are usually set within another role using _venv_path, _venv_command and _ansible_install_username but can be overridden. #venv_path: "/home/{{ ce_provision.username }}/ansible" #venv_command: /usr/bin/python3.11 -m venv - #ansible_user: deploy # user to become when creating venv + #ansible_install_username: deploy # user to become when creating venv upgrade: enabled: true # create systemd timer to auto-upgrade Ansible command: "{{ _venv_path }}/bin/python3 -m pip install --upgrade ansible" # if you set venv_path above then set it here too diff --git a/roles/debian/ansible/defaults/main.yml b/roles/debian/ansible/defaults/main.yml index 2c6108938..319bdfe5d 100644 --- a/roles/debian/ansible/defaults/main.yml +++ b/roles/debian/ansible/defaults/main.yml @@ -1,9 +1,9 @@ --- ce_ansible: - # These are usually set within another role using _venv_path, _venv_command and _ansible_user but can be overridden. + # These are usually set within another role using _venv_path, _venv_command and _ansible_install_username but can be overridden. #venv_path: "/home/{{ ce_provision.username }}/ansible" #venv_command: /usr/bin/python3.11 -m venv - #ansible_user: deploy # user to become when creating venv + #ansible_install_username: deploy # user to become when creating venv upgrade: enabled: true # create systemd timer to auto-upgrade Ansible command: "{{ _venv_path }}/bin/python3 -m pip install --upgrade ansible" # if you set venv_path above then set it here too diff --git a/roles/debian/ansible/tasks/main.yml b/roles/debian/ansible/tasks/main.yml index ead087450..5b419b787 100644 --- a/roles/debian/ansible/tasks/main.yml +++ b/roles/debian/ansible/tasks/main.yml @@ -24,7 +24,7 @@ virtualenv: "{{ ce_ansible.venv_path | default(_venv_path) }}" virtualenv_command: "{{ ce_ansible.venv_command | default(_venv_command) }}" become: true - become_user: "{{ ce_ansible.ansible_user | default(_ansible_user) }}" + become_user: "{{ ce_ansible.ansible_install_username | default(_ansible_install_username) }}" - name: Install Ansible. ansible.builtin.pip: @@ -33,7 +33,7 @@ virtualenv: "{{ ce_ansible.venv_path | default(_venv_path) }}" virtualenv_command: "{{ ce_ansible.venv_command | default(_venv_command) }}" become: true - become_user: "{{ ce_ansible.ansible_user | default(_ansible_user) }}" + become_user: "{{ ce_ansible.ansible_install_username | default(_ansible_install_username) }}" - name: Install linters. ansible.builtin.pip: @@ -44,7 +44,7 @@ virtualenv_command: "{{ ce_ansible.venv_command | default(_venv_command) }}" when: ce_ansible.linters.enabled become: true - become_user: "{{ ce_ansible.ansible_user | default(_ansible_user) }}" + become_user: "{{ ce_ansible.ansible_install_username | default(_ansible_install_username) }}" - name: Add the venv to $PATH using profile.d. ansible.builtin.copy: diff --git a/roles/debian/ce_deploy/README.md b/roles/debian/ce_deploy/README.md index d829d089f..b9f3e983d 100644 --- a/roles/debian/ce_deploy/README.md +++ b/roles/debian/ce_deploy/README.md @@ -14,7 +14,7 @@ ce_deploy: # Location of Ansible installation and components. venv_path: "/home/{{ _ce_deploy.username }}/ansible" venv_command: /usr/bin/python3 -m venv - ansible_user: "{{ _ce_deploy.username }}" + ansible_install_username: "{{ _ce_deploy.username }}" # Other ce-deploy settings. new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user key_name: id_rsa.pub # existing users may have a key of a different name diff --git a/roles/debian/ce_deploy/defaults/main.yml b/roles/debian/ce_deploy/defaults/main.yml index 276780b3b..53ee6f8ed 100644 --- a/roles/debian/ce_deploy/defaults/main.yml +++ b/roles/debian/ce_deploy/defaults/main.yml @@ -6,7 +6,7 @@ ce_deploy: # Location of Ansible installation and components. venv_path: "/home/{{ _ce_deploy.username }}/ansible" venv_command: /usr/bin/python3 -m venv - ansible_user: "{{ _ce_deploy.username }}" + ansible_install_username: "{{ _ce_deploy.username }}" # Other ce-deploy settings. new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user key_name: id_rsa.pub # existing users may have a key of a different name diff --git a/roles/debian/ce_deploy/tasks/main.yml b/roles/debian/ce_deploy/tasks/main.yml index f047afacc..90143cdb5 100644 --- a/roles/debian/ce_deploy/tasks/main.yml +++ b/roles/debian/ce_deploy/tasks/main.yml @@ -132,7 +132,7 @@ ansible.builtin.set_fact: _venv_path: "{{ ce_deploy.venv_path }}" _venv_command: "{{ ce_deploy.venv_command }}" - _ansible_user: "{{ ce_deploy.ansible_user }}" + _ansible_install_username: "{{ ce_deploy.ansible_install_username }}" - name: Install Ansible. ansible.builtin.include_role: diff --git a/roles/debian/ce_provision/README.md b/roles/debian/ce_provision/README.md index 001d8fad6..305ec1894 100644 --- a/roles/debian/ce_provision/README.md +++ b/roles/debian/ce_provision/README.md @@ -15,7 +15,7 @@ ce_provision: # Location of Ansible installation and components. venv_path: "/home/{{ _ce_provision.username }}/ansible" venv_command: /usr/bin/python3 -m venv - ansible_user: "{{ _ce_provision.username }}" + ansible_install_username: "{{ _ce_provision.username }}" # Other ce-provision settings. username: "{{ _ce_provision.username }}" new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user diff --git a/roles/debian/ce_provision/defaults/main.yml b/roles/debian/ce_provision/defaults/main.yml index 35da59995..d5e33b279 100644 --- a/roles/debian/ce_provision/defaults/main.yml +++ b/roles/debian/ce_provision/defaults/main.yml @@ -7,7 +7,7 @@ ce_provision: # Location of Ansible installation and components. venv_path: "/home/{{ _ce_provision.username }}/ansible" venv_command: /usr/bin/python3 -m venv - ansible_user: "{{ _ce_provision.username }}" + ansible_install_username: "{{ _ce_provision.username }}" # Other ce-provision settings. username: "{{ _ce_provision.username }}" new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user diff --git a/roles/debian/ce_provision/tasks/main.yml b/roles/debian/ce_provision/tasks/main.yml index 62662a1f5..ba1be5773 100644 --- a/roles/debian/ce_provision/tasks/main.yml +++ b/roles/debian/ce_provision/tasks/main.yml @@ -140,7 +140,7 @@ ansible.builtin.set_fact: _venv_path: "{{ ce_provision.venv_path }}" _venv_command: "{{ ce_provision.venv_command }}" - _ansible_user: "{{ ce_provision.ansible_user }}" + _ansible_install_username: "{{ ce_provision.ansible_install_username }}" - name: Install Ansible. ansible.builtin.include_role: From c6af4c0f7ea36b38a3c0417dca1abd747da3c9dc Mon Sep 17 00:00:00 2001 From: gregharvey Date: Wed, 10 Jan 2024 13:10:59 +0100 Subject: [PATCH 11/33] _ansible_ANYTHING is reserved, using _install_username instead. --- docs/roles/debian/ansible.md | 4 ++-- docs/roles/debian/ce_deploy.md | 2 +- docs/roles/debian/ce_provision.md | 2 +- roles/debian/ansible/README.md | 4 ++-- roles/debian/ansible/defaults/main.yml | 4 ++-- roles/debian/ansible/tasks/main.yml | 6 +++--- roles/debian/ce_deploy/README.md | 2 +- roles/debian/ce_deploy/defaults/main.yml | 2 +- roles/debian/ce_deploy/tasks/main.yml | 2 +- roles/debian/ce_provision/README.md | 2 +- roles/debian/ce_provision/defaults/main.yml | 2 +- roles/debian/ce_provision/tasks/main.yml | 2 +- 12 files changed, 17 insertions(+), 17 deletions(-) diff --git a/docs/roles/debian/ansible.md b/docs/roles/debian/ansible.md index faad26b84..b7dfe0dbc 100644 --- a/docs/roles/debian/ansible.md +++ b/docs/roles/debian/ansible.md @@ -11,10 +11,10 @@ Note, it is vitally important that Ansible is *not* installed via `apt` or `pip` ```yaml --- ce_ansible: - # These are usually set within another role using _venv_path, _venv_command and _ansible_install_username but can be overridden. + # These are usually set within another role using _venv_path, _venv_command and _install_username but can be overridden. #venv_path: "/home/{{ ce_provision.username }}/ansible" #venv_command: /usr/bin/python3.11 -m venv - #ansible_install_username: deploy # user to become when creating venv + #install_username: deploy # user to become when creating venv upgrade: enabled: true # create systemd timer to auto-upgrade Ansible command: "{{ _venv_path }}/bin/python3 -m pip install --upgrade ansible" # if you set venv_path above then set it here too diff --git a/docs/roles/debian/ce_deploy.md b/docs/roles/debian/ce_deploy.md index b9f3e983d..d020663ab 100644 --- a/docs/roles/debian/ce_deploy.md +++ b/docs/roles/debian/ce_deploy.md @@ -14,7 +14,7 @@ ce_deploy: # Location of Ansible installation and components. venv_path: "/home/{{ _ce_deploy.username }}/ansible" venv_command: /usr/bin/python3 -m venv - ansible_install_username: "{{ _ce_deploy.username }}" + install_username: "{{ _ce_deploy.username }}" # Other ce-deploy settings. new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user key_name: id_rsa.pub # existing users may have a key of a different name diff --git a/docs/roles/debian/ce_provision.md b/docs/roles/debian/ce_provision.md index 305ec1894..e8f83ea13 100644 --- a/docs/roles/debian/ce_provision.md +++ b/docs/roles/debian/ce_provision.md @@ -15,7 +15,7 @@ ce_provision: # Location of Ansible installation and components. venv_path: "/home/{{ _ce_provision.username }}/ansible" venv_command: /usr/bin/python3 -m venv - ansible_install_username: "{{ _ce_provision.username }}" + install_username: "{{ _ce_provision.username }}" # Other ce-provision settings. username: "{{ _ce_provision.username }}" new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user diff --git a/roles/debian/ansible/README.md b/roles/debian/ansible/README.md index faad26b84..b7dfe0dbc 100644 --- a/roles/debian/ansible/README.md +++ b/roles/debian/ansible/README.md @@ -11,10 +11,10 @@ Note, it is vitally important that Ansible is *not* installed via `apt` or `pip` ```yaml --- ce_ansible: - # These are usually set within another role using _venv_path, _venv_command and _ansible_install_username but can be overridden. + # These are usually set within another role using _venv_path, _venv_command and _install_username but can be overridden. #venv_path: "/home/{{ ce_provision.username }}/ansible" #venv_command: /usr/bin/python3.11 -m venv - #ansible_install_username: deploy # user to become when creating venv + #install_username: deploy # user to become when creating venv upgrade: enabled: true # create systemd timer to auto-upgrade Ansible command: "{{ _venv_path }}/bin/python3 -m pip install --upgrade ansible" # if you set venv_path above then set it here too diff --git a/roles/debian/ansible/defaults/main.yml b/roles/debian/ansible/defaults/main.yml index 319bdfe5d..f58a026b0 100644 --- a/roles/debian/ansible/defaults/main.yml +++ b/roles/debian/ansible/defaults/main.yml @@ -1,9 +1,9 @@ --- ce_ansible: - # These are usually set within another role using _venv_path, _venv_command and _ansible_install_username but can be overridden. + # These are usually set within another role using _venv_path, _venv_command and _install_username but can be overridden. #venv_path: "/home/{{ ce_provision.username }}/ansible" #venv_command: /usr/bin/python3.11 -m venv - #ansible_install_username: deploy # user to become when creating venv + #install_username: deploy # user to become when creating venv upgrade: enabled: true # create systemd timer to auto-upgrade Ansible command: "{{ _venv_path }}/bin/python3 -m pip install --upgrade ansible" # if you set venv_path above then set it here too diff --git a/roles/debian/ansible/tasks/main.yml b/roles/debian/ansible/tasks/main.yml index 5b419b787..4aac433a9 100644 --- a/roles/debian/ansible/tasks/main.yml +++ b/roles/debian/ansible/tasks/main.yml @@ -24,7 +24,7 @@ virtualenv: "{{ ce_ansible.venv_path | default(_venv_path) }}" virtualenv_command: "{{ ce_ansible.venv_command | default(_venv_command) }}" become: true - become_user: "{{ ce_ansible.ansible_install_username | default(_ansible_install_username) }}" + become_user: "{{ ce_ansible.install_username | default(_install_username) }}" - name: Install Ansible. ansible.builtin.pip: @@ -33,7 +33,7 @@ virtualenv: "{{ ce_ansible.venv_path | default(_venv_path) }}" virtualenv_command: "{{ ce_ansible.venv_command | default(_venv_command) }}" become: true - become_user: "{{ ce_ansible.ansible_install_username | default(_ansible_install_username) }}" + become_user: "{{ ce_ansible.install_username | default(_install_username) }}" - name: Install linters. ansible.builtin.pip: @@ -44,7 +44,7 @@ virtualenv_command: "{{ ce_ansible.venv_command | default(_venv_command) }}" when: ce_ansible.linters.enabled become: true - become_user: "{{ ce_ansible.ansible_install_username | default(_ansible_install_username) }}" + become_user: "{{ ce_ansible.install_username | default(_install_username) }}" - name: Add the venv to $PATH using profile.d. ansible.builtin.copy: diff --git a/roles/debian/ce_deploy/README.md b/roles/debian/ce_deploy/README.md index b9f3e983d..d020663ab 100644 --- a/roles/debian/ce_deploy/README.md +++ b/roles/debian/ce_deploy/README.md @@ -14,7 +14,7 @@ ce_deploy: # Location of Ansible installation and components. venv_path: "/home/{{ _ce_deploy.username }}/ansible" venv_command: /usr/bin/python3 -m venv - ansible_install_username: "{{ _ce_deploy.username }}" + install_username: "{{ _ce_deploy.username }}" # Other ce-deploy settings. new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user key_name: id_rsa.pub # existing users may have a key of a different name diff --git a/roles/debian/ce_deploy/defaults/main.yml b/roles/debian/ce_deploy/defaults/main.yml index 53ee6f8ed..52bd86415 100644 --- a/roles/debian/ce_deploy/defaults/main.yml +++ b/roles/debian/ce_deploy/defaults/main.yml @@ -6,7 +6,7 @@ ce_deploy: # Location of Ansible installation and components. venv_path: "/home/{{ _ce_deploy.username }}/ansible" venv_command: /usr/bin/python3 -m venv - ansible_install_username: "{{ _ce_deploy.username }}" + install_username: "{{ _ce_deploy.username }}" # Other ce-deploy settings. new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user key_name: id_rsa.pub # existing users may have a key of a different name diff --git a/roles/debian/ce_deploy/tasks/main.yml b/roles/debian/ce_deploy/tasks/main.yml index 90143cdb5..a9d607723 100644 --- a/roles/debian/ce_deploy/tasks/main.yml +++ b/roles/debian/ce_deploy/tasks/main.yml @@ -132,7 +132,7 @@ ansible.builtin.set_fact: _venv_path: "{{ ce_deploy.venv_path }}" _venv_command: "{{ ce_deploy.venv_command }}" - _ansible_install_username: "{{ ce_deploy.ansible_install_username }}" + _install_username: "{{ ce_deploy.install_username }}" - name: Install Ansible. ansible.builtin.include_role: diff --git a/roles/debian/ce_provision/README.md b/roles/debian/ce_provision/README.md index 305ec1894..e8f83ea13 100644 --- a/roles/debian/ce_provision/README.md +++ b/roles/debian/ce_provision/README.md @@ -15,7 +15,7 @@ ce_provision: # Location of Ansible installation and components. venv_path: "/home/{{ _ce_provision.username }}/ansible" venv_command: /usr/bin/python3 -m venv - ansible_install_username: "{{ _ce_provision.username }}" + install_username: "{{ _ce_provision.username }}" # Other ce-provision settings. username: "{{ _ce_provision.username }}" new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user diff --git a/roles/debian/ce_provision/defaults/main.yml b/roles/debian/ce_provision/defaults/main.yml index d5e33b279..4c266e3e0 100644 --- a/roles/debian/ce_provision/defaults/main.yml +++ b/roles/debian/ce_provision/defaults/main.yml @@ -7,7 +7,7 @@ ce_provision: # Location of Ansible installation and components. venv_path: "/home/{{ _ce_provision.username }}/ansible" venv_command: /usr/bin/python3 -m venv - ansible_install_username: "{{ _ce_provision.username }}" + install_username: "{{ _ce_provision.username }}" # Other ce-provision settings. username: "{{ _ce_provision.username }}" new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user diff --git a/roles/debian/ce_provision/tasks/main.yml b/roles/debian/ce_provision/tasks/main.yml index ba1be5773..ca59d5928 100644 --- a/roles/debian/ce_provision/tasks/main.yml +++ b/roles/debian/ce_provision/tasks/main.yml @@ -140,7 +140,7 @@ ansible.builtin.set_fact: _venv_path: "{{ ce_provision.venv_path }}" _venv_command: "{{ ce_provision.venv_command }}" - _ansible_install_username: "{{ ce_provision.ansible_install_username }}" + _install_username: "{{ ce_provision.install_username }}" - name: Install Ansible. ansible.builtin.include_role: From b3026765ac2516ef1df8c48783e43ff41bde278d Mon Sep 17 00:00:00 2001 From: gregharvey Date: Wed, 10 Jan 2024 18:00:23 +0100 Subject: [PATCH 12/33] python_boto role also needs the username set in the calling role. --- roles/debian/python_boto/defaults/main.yml | 3 ++- roles/debian/python_boto/tasks/main.yml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/roles/debian/python_boto/defaults/main.yml b/roles/debian/python_boto/defaults/main.yml index 489e33ef6..69daa3a62 100644 --- a/roles/debian/python_boto/defaults/main.yml +++ b/roles/debian/python_boto/defaults/main.yml @@ -2,4 +2,5 @@ python_boto: boto3_version: "" # version string, e.g. "1.22.13" - empty string means latest # These are usually set within another role using _venv_path and _venv_command but can be overridden. #venv_path: /path/to/venv - #venv_command: /usr/bin/python3.11 -m venv \ No newline at end of file + #venv_command: /usr/bin/python3.11 -m venv + #install_username: deploy # user to become when creating venv diff --git a/roles/debian/python_boto/tasks/main.yml b/roles/debian/python_boto/tasks/main.yml index 8b989f63c..c89cc37e6 100644 --- a/roles/debian/python_boto/tasks/main.yml +++ b/roles/debian/python_boto/tasks/main.yml @@ -15,4 +15,4 @@ virtualenv: "{{ python_boto.venv_path | default(_venv_path) }}" virtualenv_command: "{{ python_boto.venv_command | default(_venv_command) }}" become: true - become_user: "{{ ce_provision.username }}" + become_user: "{{ python_boto.install_username | default(_install_username) }}" From 2fe29a1457002b496e3cf17b341154924d4b2d49 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Wed, 10 Jan 2024 18:00:48 +0100 Subject: [PATCH 13/33] Updating python_boto docs. --- docs/roles/debian/python_boto.md | 2 ++ roles/debian/python_boto/README.md | 2 ++ 2 files changed, 4 insertions(+) diff --git a/docs/roles/debian/python_boto.md b/docs/roles/debian/python_boto.md index ee3e99a52..268af9614 100644 --- a/docs/roles/debian/python_boto.md +++ b/docs/roles/debian/python_boto.md @@ -12,6 +12,8 @@ python_boto: # These are usually set within another role using _venv_path and _venv_command but can be overridden. #venv_path: /path/to/venv #venv_command: /usr/bin/python3.11 -m venv + #install_username: deploy # user to become when creating venv + ``` diff --git a/roles/debian/python_boto/README.md b/roles/debian/python_boto/README.md index ee3e99a52..268af9614 100644 --- a/roles/debian/python_boto/README.md +++ b/roles/debian/python_boto/README.md @@ -12,6 +12,8 @@ python_boto: # These are usually set within another role using _venv_path and _venv_command but can be overridden. #venv_path: /path/to/venv #venv_command: /usr/bin/python3.11 -m venv + #install_username: deploy # user to become when creating venv + ``` From 32f3ced402d3020afbc2d5aa3fae93a0bfca8729 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Thu, 11 Jan 2024 14:21:52 +0100 Subject: [PATCH 14/33] Making profile.d loading more robust. --- scripts/_common.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/scripts/_common.sh b/scripts/_common.sh index 0da872d6f..a1aae6923 100755 --- a/scripts/_common.sh +++ b/scripts/_common.sh @@ -34,10 +34,12 @@ if [ ! -d "$ANSIBLE_DATA_DIR" ]; then mkdir "$ANSIBLE_DATA_DIR" fi # Load the contents of profile.d in case we added items to $PATH there. -for f in /etc/profile.d/*; do -# shellcheck source=/dev/null - . "$f" -done +if [ -n "$(ls -A /etc/profile.d)" ]; then + for f in /etc/profile.d/*; do + # shellcheck source=/dev/null + . "$f" + done +fi # Parse options arguments. parse_options(){ while [ "${1:-}" ]; do From 1f8d80e99ee28f40c41d60e976fbbfbf7c262698 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Thu, 11 Jan 2024 14:22:34 +0100 Subject: [PATCH 15/33] Also pip removing ansible-core and trying with pip and pip3 to cover all bases. --- roles/debian/ansible/tasks/main.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/roles/debian/ansible/tasks/main.yml b/roles/debian/ansible/tasks/main.yml index 4aac433a9..7ea4f100d 100644 --- a/roles/debian/ansible/tasks/main.yml +++ b/roles/debian/ansible/tasks/main.yml @@ -4,12 +4,17 @@ pkg: ansible state: absent -- name: Ensure Ansible is not installed globally with pip3 on older versions of Debian. +- name: Ensure Ansible is not installed globally with pip or pip3 on older versions of Debian. ansible.builtin.pip: - name: ansible + name: + - ansible + - ansible-core state: absent - executable: pip3 + executable: "{{ item }}" when: ansible_distribution_major_version | int < 12 + with_items: + - pip + - pip3 failed_when: false # don't stop the build if there's no system pip - name: Set up Python packages. From ff2803f77418a668031bff0b82c52214126dc506 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Thu, 11 Jan 2024 14:23:27 +0100 Subject: [PATCH 16/33] Updating bad AWS SG role var namespacing in other roles. --- .../aws_ec2_autoscale_cluster/tasks/main.yml | 18 ++++++++++-------- roles/aws/aws_ec2_with_eip/tasks/main.yml | 9 +++++---- roles/aws/aws_efs/tasks/main.yml | 9 +++++---- roles/aws/aws_elasticache/tasks/main.yml | 9 +++++---- roles/aws/aws_elb/tasks/main.yml | 9 +++++---- roles/aws/aws_opensearch/tasks/main.yml | 9 +++++---- roles/aws/aws_rds/tasks/main.yml | 9 +++++---- roles/debian/gitlab_runner/tasks/main.yml | 11 ++++++----- 8 files changed, 46 insertions(+), 37 deletions(-) diff --git a/roles/aws/aws_ec2_autoscale_cluster/tasks/main.yml b/roles/aws/aws_ec2_autoscale_cluster/tasks/main.yml index 70597d699..611ba383c 100644 --- a/roles/aws/aws_ec2_autoscale_cluster/tasks/main.yml +++ b/roles/aws/aws_ec2_autoscale_cluster/tasks/main.yml @@ -379,10 +379,11 @@ ansible.builtin.include_role: name: aws/aws_security_groups vars: - profile: "{{ aws_ec2_autoscale_cluster.aws_profile }}" - region: "{{ aws_ec2_autoscale_cluster.region }}" - group_names: "{{ aws_ec2_autoscale_cluster.alb_security_groups }}" - return_type: ids + aws_security_groups: + profile: "{{ aws_ec2_autoscale_cluster.aws_profile }}" + region: "{{ aws_ec2_autoscale_cluster.region }}" + group_names: "{{ aws_ec2_autoscale_cluster.alb_security_groups }}" + return_type: ids when: - aws_ec2_autoscale_cluster.alb_security_groups | length > 0 - aws_ec2_autoscale_cluster.create_elb @@ -425,10 +426,11 @@ ansible.builtin.include_role: name: aws/aws_security_groups vars: - profile: "{{ aws_ec2_autoscale_cluster.aws_profile }}" - region: "{{ aws_ec2_autoscale_cluster.region }}" - group_names: "{{ aws_ec2_autoscale_cluster.cluster_security_groups }}" - return_type: ids + aws_security_groups: + profile: "{{ aws_ec2_autoscale_cluster.aws_profile }}" + region: "{{ aws_ec2_autoscale_cluster.region }}" + group_names: "{{ aws_ec2_autoscale_cluster.cluster_security_groups }}" + return_type: ids when: - aws_ec2_autoscale_cluster.cluster_security_groups | length > 0 - aws_ec2_autoscale_cluster.asg_refresh diff --git a/roles/aws/aws_ec2_with_eip/tasks/main.yml b/roles/aws/aws_ec2_with_eip/tasks/main.yml index 06aaec917..fea2c4ecb 100644 --- a/roles/aws/aws_ec2_with_eip/tasks/main.yml +++ b/roles/aws/aws_ec2_with_eip/tasks/main.yml @@ -95,10 +95,11 @@ ansible.builtin.include_role: name: aws/aws_security_groups vars: - profile: "{{ aws_ec2_with_eip.aws_profile }}" - region: "{{ aws_ec2_with_eip.region }}" - group_names: "{{ aws_ec2_with_eip.security_groups }}" - return_type: ids + aws_security_groups: + profile: "{{ aws_ec2_with_eip.aws_profile }}" + region: "{{ aws_ec2_with_eip.region }}" + group_names: "{{ aws_ec2_with_eip.security_groups }}" + return_type: ids when: aws_ec2_with_eip.security_groups | length > 0 # Do not create an instance if _aws_hostname is not an EC2 generated address unless `force: true` diff --git a/roles/aws/aws_efs/tasks/main.yml b/roles/aws/aws_efs/tasks/main.yml index 6e07019b5..b7351ebd8 100644 --- a/roles/aws/aws_efs/tasks/main.yml +++ b/roles/aws/aws_efs/tasks/main.yml @@ -3,10 +3,11 @@ ansible.builtin.include_role: name: aws/aws_security_groups vars: - profile: "{{ aws_efs.aws_profile }}" - region: "{{ aws_efs.region }}" - group_names: "{{ aws_efs.security_groups }}" - return_type: ids + aws_security_groups: + profile: "{{ aws_efs.aws_profile }}" + region: "{{ aws_efs.region }}" + group_names: "{{ aws_efs.security_groups }}" + return_type: ids when: aws_efs.security_groups | length > 0 - name: Reset subnet ids info diff --git a/roles/aws/aws_elasticache/tasks/main.yml b/roles/aws/aws_elasticache/tasks/main.yml index 83c64efd8..7c75d36eb 100644 --- a/roles/aws/aws_elasticache/tasks/main.yml +++ b/roles/aws/aws_elasticache/tasks/main.yml @@ -3,10 +3,11 @@ ansible.builtin.include_role: name: aws/aws_security_groups vars: - profile: "{{ aws_elasticache.aws_profile }}" - region: "{{ aws_elasticache.region }}" - group_names: "{{ aws_elasticache.elasticache_security_groups }}" - return_type: ids + aws_security_groups: + profile: "{{ aws_elasticache.aws_profile }}" + region: "{{ aws_elasticache.region }}" + group_names: "{{ aws_elasticache.elasticache_security_groups }}" + return_type: ids when: aws_elasticache.elasticache_security_groups | length > 0 - name: Create ElastiCache subnet group. diff --git a/roles/aws/aws_elb/tasks/main.yml b/roles/aws/aws_elb/tasks/main.yml index 4c4416338..a47f372ad 100644 --- a/roles/aws/aws_elb/tasks/main.yml +++ b/roles/aws/aws_elb/tasks/main.yml @@ -113,10 +113,11 @@ ansible.builtin.include_role: name: aws/aws_security_groups vars: - profile: "{{ aws_elb.aws_profile }}" - region: "{{ aws_elb.region }}" - group_names: "{{ aws_elb.elb_security_groups }}" - return_type: ids + aws_security_groups: + profile: "{{ aws_elb.aws_profile }}" + region: "{{ aws_elb.region }}" + group_names: "{{ aws_elb.elb_security_groups }}" + return_type: ids when: aws_elb.elb_security_groups | length > 0 - name: Create the ALB. diff --git a/roles/aws/aws_opensearch/tasks/main.yml b/roles/aws/aws_opensearch/tasks/main.yml index f43b79bb0..39368148a 100644 --- a/roles/aws/aws_opensearch/tasks/main.yml +++ b/roles/aws/aws_opensearch/tasks/main.yml @@ -4,10 +4,11 @@ ansible.builtin.include_role: name: aws/aws_security_groups vars: - profile: "{{ aws_opensearch.aws_profile }}" - region: "{{ aws_opensearch.region }}" - group_names: "{{ aws_opensearch.security_groups }}" - return_type: ids + aws_security_groups: + profile: "{{ aws_opensearch.aws_profile }}" + region: "{{ aws_opensearch.region }}" + group_names: "{{ aws_opensearch.security_groups }}" + return_type: ids when: aws_opensearch.security_groups | length > 0 - name: Get the current AWS account caller identity information diff --git a/roles/aws/aws_rds/tasks/main.yml b/roles/aws/aws_rds/tasks/main.yml index 416dd30b4..85301323d 100644 --- a/roles/aws/aws_rds/tasks/main.yml +++ b/roles/aws/aws_rds/tasks/main.yml @@ -67,10 +67,11 @@ ansible.builtin.include_role: name: aws/aws_security_groups vars: - profile: "{{ aws_rds.aws_profile }}" - region: "{{ aws_rds.region }}" - group_names: "{{ aws_rds.security_groups }}" - return_type: ids + aws_security_groups: + profile: "{{ aws_rds.aws_profile }}" + region: "{{ aws_rds.region }}" + group_names: "{{ aws_rds.security_groups }}" + return_type: ids when: aws_rds.security_groups | length > 0 - name: Create RDS instance diff --git a/roles/debian/gitlab_runner/tasks/main.yml b/roles/debian/gitlab_runner/tasks/main.yml index 685b0bfc2..47b32ac4c 100644 --- a/roles/debian/gitlab_runner/tasks/main.yml +++ b/roles/debian/gitlab_runner/tasks/main.yml @@ -42,11 +42,12 @@ apply: delegate_to: localhost # this is how you delegate the include_role module vars: - profile: "{{ gitlab_runner.fargate.profile }}" - region: "{{ gitlab_runner.fargate.region }}" - group_names: - - "{{ gitlab_runner.fargate.security_group }}" - return_type: ids + aws_security_groups: + profile: "{{ gitlab_runner.fargate.profile }}" + region: "{{ gitlab_runner.fargate.region }}" + group_names: + - "{{ gitlab_runner.fargate.security_group }}" + return_type: ids when: gitlab_runner.install_fargate - name: Create the Fargate driver directory if it does not exist. From 09663980f70358fe135ff16488218e0b16814da4 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Thu, 11 Jan 2024 20:37:14 +0100 Subject: [PATCH 17/33] Refactoring how we handle python3-pip. --- roles/debian/ldap_server/meta/main.yml | 3 --- roles/debian/ldap_server/tasks/main.yml | 13 +++++++++++++ roles/debian/mysql_client/meta/main.yml | 3 --- roles/debian/mysql_client/tasks/main.yml | 13 +++++++++++++ roles/debian/nginx/meta/main.yml | 3 --- roles/debian/nginx/tasks/main.yml | 13 +++++++++++++ roles/debian/python_common/defaults/main.yml | 1 - 7 files changed, 39 insertions(+), 10 deletions(-) delete mode 100644 roles/debian/ldap_server/meta/main.yml delete mode 100644 roles/debian/mysql_client/meta/main.yml delete mode 100644 roles/debian/nginx/meta/main.yml diff --git a/roles/debian/ldap_server/meta/main.yml b/roles/debian/ldap_server/meta/main.yml deleted file mode 100644 index 551567003..000000000 --- a/roles/debian/ldap_server/meta/main.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -dependencies: - - role: debian/python_common diff --git a/roles/debian/ldap_server/tasks/main.yml b/roles/debian/ldap_server/tasks/main.yml index 9a773da58..bf2c0d3d1 100644 --- a/roles/debian/ldap_server/tasks/main.yml +++ b/roles/debian/ldap_server/tasks/main.yml @@ -14,6 +14,19 @@ purge: true when: ldap_server.slapd.purge +- name: Set up Python packages. + ansible.builtin.include_role: + name: debian/python_common + vars: + python_common: + remove_packages: + - python-pip + - python3-yaml + - python-botocore + - python-urllib3 + install_packages: + - python3-pip + - name: Ensure LDAP and python-ldap and dependencies are installed. ansible.builtin.apt: pkg: ["slapd", "ldapscripts", "libldap2-dev"] diff --git a/roles/debian/mysql_client/meta/main.yml b/roles/debian/mysql_client/meta/main.yml deleted file mode 100644 index 551567003..000000000 --- a/roles/debian/mysql_client/meta/main.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -dependencies: - - role: debian/python_common diff --git a/roles/debian/mysql_client/tasks/main.yml b/roles/debian/mysql_client/tasks/main.yml index d17123205..2ecca067e 100644 --- a/roles/debian/mysql_client/tasks/main.yml +++ b/roles/debian/mysql_client/tasks/main.yml @@ -6,6 +6,19 @@ update_cache: true cache_valid_time: 240 +- name: Set up Python packages. + ansible.builtin.include_role: + name: debian/python_common + vars: + python_common: + remove_packages: + - python-pip + - python3-yaml + - python-botocore + - python-urllib3 + install_packages: + - python3-pip + - name: Ensure PyMySQL is installed with pip3. ansible.builtin.pip: name: pymysql diff --git a/roles/debian/nginx/meta/main.yml b/roles/debian/nginx/meta/main.yml deleted file mode 100644 index 551567003..000000000 --- a/roles/debian/nginx/meta/main.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -dependencies: - - role: debian/python_common diff --git a/roles/debian/nginx/tasks/main.yml b/roles/debian/nginx/tasks/main.yml index 60a8c4246..0388ed007 100644 --- a/roles/debian/nginx/tasks/main.yml +++ b/roles/debian/nginx/tasks/main.yml @@ -6,6 +6,19 @@ update_cache: true cache_valid_time: 240 +- name: Set up Python packages. + ansible.builtin.include_role: + name: debian/python_common + vars: + python_common: + remove_packages: + - python-pip + - python3-yaml + - python-botocore + - python-urllib3 + install_packages: + - python3-pip + - name: Ensure passlib is installed with pip3. ansible.builtin.pip: name: passlib diff --git a/roles/debian/python_common/defaults/main.yml b/roles/debian/python_common/defaults/main.yml index afb3840df..59c3c7e72 100644 --- a/roles/debian/python_common/defaults/main.yml +++ b/roles/debian/python_common/defaults/main.yml @@ -8,5 +8,4 @@ python_common: install_packages: - python3-distutils - python3-venv - - python3-pip - cloud-init # package can get removed with python3-yaml but we need it for auto-scale \ No newline at end of file From 6b2b69b137c08153e459f323e97f5cd75ba4455e Mon Sep 17 00:00:00 2001 From: gregharvey Date: Fri, 12 Jan 2024 09:42:08 +0100 Subject: [PATCH 18/33] Allow passing in of the Python interpreter to Ansible. --- scripts/_common.sh | 11 ++++++++++- scripts/provision.sh | 1 + 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/scripts/_common.sh b/scripts/_common.sh index a1aae6923..da5277ef9 100755 --- a/scripts/_common.sh +++ b/scripts/_common.sh @@ -12,6 +12,7 @@ TARGET_PROVISION_BRANCH="" ANSIBLE_EXTRA_VARS="" ANSIBLE_DEFAULT_EXTRA_VARS="" ANSIBLE_PATH="" +PYTHON_INTERPRETER="" BUILD_WORKSPACE="" BUILD_WORKSPACE_BASE="$OWN_DIR/build" BUILD_ID="" @@ -101,6 +102,10 @@ parse_options(){ shift ANSIBLE_PATH="$1" ;; + "--python-interpreter") + shift + PYTHON_INTERPRETER="$1" + ;; *) usage exit 1 @@ -123,7 +128,11 @@ get_build_workspace(){ # Common extra-vars to pass to Ansible. get_ansible_defaults_vars(){ get_build_id - ANSIBLE_DEFAULT_EXTRA_VARS="{_ce_provision_base_dir: $OWN_DIR, _ce_provision_build_dir: $BUILD_WORKSPACE, _ce_provision_build_tmp_dir: $BUILD_TMP_DIR, _ce_provision_data_dir: $ANSIBLE_DATA_DIR, _ce_provision_build_id: $BUILD_ID, _ce_provision_force_play: $FORCE_PLAY, target_branch: $TARGET_PROVISION_BRANCH}" + if [ -n "$PYTHON_INTERPRETER" ]; then + ANSIBLE_DEFAULT_EXTRA_VARS="{ansible_python_interpreter: $PYTHON_INTERPRETER, _ce_provision_base_dir: $OWN_DIR, _ce_provision_build_dir: $BUILD_WORKSPACE, _ce_provision_build_tmp_dir: $BUILD_TMP_DIR, _ce_provision_data_dir: $ANSIBLE_DATA_DIR, _ce_provision_build_id: $BUILD_ID, _ce_provision_force_play: $FORCE_PLAY, target_branch: $TARGET_PROVISION_BRANCH}" + else + ANSIBLE_DEFAULT_EXTRA_VARS="{_ce_provision_base_dir: $OWN_DIR, _ce_provision_build_dir: $BUILD_WORKSPACE, _ce_provision_build_tmp_dir: $BUILD_TMP_DIR, _ce_provision_data_dir: $ANSIBLE_DATA_DIR, _ce_provision_build_id: $BUILD_ID, _ce_provision_force_play: $FORCE_PLAY, target_branch: $TARGET_PROVISION_BRANCH}" + fi } # Clone our target repo. diff --git a/scripts/provision.sh b/scripts/provision.sh index 92d2e0265..fda465e05 100755 --- a/scripts/provision.sh +++ b/scripts/provision.sh @@ -14,6 +14,7 @@ usage(){ echo 'Available options:' echo '--ansible-extra-vars: Variable to pass as --extra-vars arguments to ansible-playbook. Make sure to escape them properly.' echo '--ansible-path: Pass the path to the directory containing the Ansible binaries if you are not using the version of Ansible in PATH.' + echo '--python-interpreter: When using Python virtual environments Ansible may not correctly determine the Python interpreter, use this to set it manually.' echo '--workspace: Local existing clone of the repo/branch (if your deployment tool already has one). This will skip the cloning/fetching of the repo.' echo '--user: Linux user executing the script (defaults to controller).' echo '--absolute-playbook-path: Prevent prepending of the workspace path to the playbook path when Ansible is called.' From 3c1d06c9cfdbc7e86a8125966cfc3107ac8e9e0b Mon Sep 17 00:00:00 2001 From: gregharvey Date: Mon, 15 Jan 2024 15:58:21 +0100 Subject: [PATCH 19/33] Updating the packages server for CE. --- roles/debian/pam_linotp/tasks/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/debian/pam_linotp/tasks/main.yml b/roles/debian/pam_linotp/tasks/main.yml index d4736a2e6..d7fe822a2 100644 --- a/roles/debian/pam_linotp/tasks/main.yml +++ b/roles/debian/pam_linotp/tasks/main.yml @@ -1,13 +1,13 @@ --- - name: Add key for LinOTP repository. ansible.builtin.apt_key: - keyserver: http://packages.codeenigma.com/debian/codeenigma.pub - id: A344A0826FD987C6 + keyserver: https://packages.codeenigma.net/debian/codeenigma.pub + id: 77AFCA1C45124FBB2EFCD7267DC594C08E0497D4 state: present - name: Add repository for LinOTP. ansible.builtin.apt_repository: - repo: "deb http://packages.codeenigma.com/debian buster main" + repo: "deb https://packages.codeenigma.net/debian {{ ansible_distribution_release }} main" state: present - name: Ensure libpam-linotp is installed. From 2b422a93772edbcf2aa8f5caa67481027ab694bd Mon Sep 17 00:00:00 2001 From: gregharvey Date: Tue, 16 Jan 2024 11:56:42 +0100 Subject: [PATCH 20/33] Installing Ansible in a venv on all machines. --- roles/_meta/common_base/meta/main.yml | 2 ++ roles/_meta/webserver/meta/main.yml | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/roles/_meta/common_base/meta/main.yml b/roles/_meta/common_base/meta/main.yml index 95fcebc29..1ff94fc17 100644 --- a/roles/_meta/common_base/meta/main.yml +++ b/roles/_meta/common_base/meta/main.yml @@ -4,6 +4,7 @@ dependencies: - { role: debian/apt_extra_packages, apt_extra_packages: ["dnsutils", "telnet", "atop", "sysstat"], when: ( is_local is not defined or not is_local ) } - role: debian/locales - role: debian/user_root + - role: debian/user_provision - role: debian/apt_extra_packages - { role: debian/apt_unattended_upgrades, when: ( is_local is not defined or not is_local ) } - { role: debian/hosts, when: ( is_local is not defined or not is_local ) } @@ -12,3 +13,4 @@ dependencies: - { role: debian/clamav, when: ( is_local is not defined or not is_local ) } - { role: debian/firewall_config, when: ( is_local is not defined or not is_local ) } - { role: debian/sudo_config, when: ( is_local is not defined or not is_local ) } + - { role: debian/ansible, ce_ansible: {venv_path: '/home/' + ce_provision.username + '/ansible', venv_command: '/usr/bin/python3.11 -m venv', install_username: ce_provision.username, upgrade: {enabled: true, command: '/home/' + ce_provision.username + '/ansible/bin/python3 -m pip install --upgrade ansible', on_calendar: '*-*-* 01:30:00'}, linters: {enabled: false}} } diff --git a/roles/_meta/webserver/meta/main.yml b/roles/_meta/webserver/meta/main.yml index a66530c03..00249c536 100644 --- a/roles/_meta/webserver/meta/main.yml +++ b/roles/_meta/webserver/meta/main.yml @@ -2,7 +2,6 @@ dependencies: - { role: debian/ssh_server, when: ( is_local is not defined or not is_local ) } - role: _meta/common_base - - role: debian/user_provision - role: debian/user_deploy - role: debian/mysql_client - role: debian/nodejs From 33abf0a173133b414a96f792c405e0d518a8bd72 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Tue, 16 Jan 2024 12:17:07 +0100 Subject: [PATCH 21/33] Changing common_base format for readability. --- roles/_meta/common_base/meta/main.yml | 57 ++++++++++++++++++++++----- 1 file changed, 48 insertions(+), 9 deletions(-) diff --git a/roles/_meta/common_base/meta/main.yml b/roles/_meta/common_base/meta/main.yml index 1ff94fc17..140fcf1bb 100644 --- a/roles/_meta/common_base/meta/main.yml +++ b/roles/_meta/common_base/meta/main.yml @@ -1,16 +1,55 @@ --- dependencies: - - { role: debian/apt_extra_packages, apt_extra_packages: ["rsync", "python3-apt", "acl", "gnupg", "build-essential", "cron", "bsd-mailx", "vim", "unzip", "ca-certificates"] } - - { role: debian/apt_extra_packages, apt_extra_packages: ["dnsutils", "telnet", "atop", "sysstat"], when: ( is_local is not defined or not is_local ) } + - role: debian/apt_extra_packages + apt_extra_packages: + - rsync + - python3-apt + - acl + - gnupg + - build-essential + - cron + - bsd-mailx + - vim + - unzip + - ca-certificates + - role: debian/apt_extra_packages + apt_extra_packages: + - dnsutils + - telnet + - atop + - sysstat + - tcpdump + - mtr-tiny + - htop + - ntp + - screen + - iptraf-ng + when: is_local is not defined or not is_local - role: debian/locales - role: debian/user_root - role: debian/user_provision - role: debian/apt_extra_packages - - { role: debian/apt_unattended_upgrades, when: ( is_local is not defined or not is_local ) } - - { role: debian/hosts, when: ( is_local is not defined or not is_local ) } - - { role: debian/rkhunter, when: ( is_local is not defined or not is_local ) } + - role: debian/apt_unattended_upgrades + when: is_local is not defined or not is_local + - role: debian/hosts + when: is_local is not defined or not is_local + - role: debian/rkhunter + when: is_local is not defined or not is_local - role: debian/postfix - - { role: debian/clamav, when: ( is_local is not defined or not is_local ) } - - { role: debian/firewall_config, when: ( is_local is not defined or not is_local ) } - - { role: debian/sudo_config, when: ( is_local is not defined or not is_local ) } - - { role: debian/ansible, ce_ansible: {venv_path: '/home/' + ce_provision.username + '/ansible', venv_command: '/usr/bin/python3.11 -m venv', install_username: ce_provision.username, upgrade: {enabled: true, command: '/home/' + ce_provision.username + '/ansible/bin/python3 -m pip install --upgrade ansible', on_calendar: '*-*-* 01:30:00'}, linters: {enabled: false}} } + - role: debian/clamav + when: is_local is not defined or not is_local + - role: debian/firewall_config + when: is_local is not defined or not is_local + - role: debian/sudo_config + when: is_local is not defined or not is_local + - role: debian/ansible + ce_ansible: + venv_path: "/home/{{ ce_provision.username }}/ansible" + venv_command: "/usr/bin/python3.11 -m venv" + install_username: "{{ ce_provision.username }}" + upgrade: + enabled: true + command: "/home/{{ ce_provision.username }}/ansible/bin/python3 -m pip install --upgrade ansible" + on_calendar: "*-*-* 01:30:00" + linters: + enabled: false From 394ec01472e16adef55bf5321a16dd8217730ff7 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Wed, 17 Jan 2024 19:02:48 +0100 Subject: [PATCH 22/33] No need to specify Python to the point release. --- roles/_meta/common_base/meta/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/_meta/common_base/meta/main.yml b/roles/_meta/common_base/meta/main.yml index 140fcf1bb..9e3635fa6 100644 --- a/roles/_meta/common_base/meta/main.yml +++ b/roles/_meta/common_base/meta/main.yml @@ -45,7 +45,7 @@ dependencies: - role: debian/ansible ce_ansible: venv_path: "/home/{{ ce_provision.username }}/ansible" - venv_command: "/usr/bin/python3.11 -m venv" + venv_command: "/usr/bin/python3 -m venv" install_username: "{{ ce_provision.username }}" upgrade: enabled: true From f906530099ea48eb6ce0149c5ac319800ffc42bd Mon Sep 17 00:00:00 2001 From: gregharvey Date: Wed, 17 Jan 2024 19:08:34 +0100 Subject: [PATCH 23/33] Docs update. --- docs/_Sidebar.md | 2 +- docs/roles/debian/python_common.md | 14 ++++++++------ docs/roles/debian/swap.md | 7 +++---- roles/debian/python_common/README.md | 14 ++++++++------ roles/debian/swap/README.md | 7 ++----- 5 files changed, 22 insertions(+), 22 deletions(-) diff --git a/docs/_Sidebar.md b/docs/_Sidebar.md index aa59208c8..5a5213a0b 100644 --- a/docs/_Sidebar.md +++ b/docs/_Sidebar.md @@ -76,7 +76,7 @@ - [SSHD](/roles/debian/ssh_server) - [SSL](/roles/debian/ssl) - [sudo config](/roles/debian/sudo_config) - - [General swap role to add swap space](/roles/debian/swap) + - [Swap](/roles/debian/swap) - [varnish-config](/roles/debian/varnish_config) - [wazuh](/roles/debian/wazuh) - [Init role](/roles/_init) diff --git a/docs/roles/debian/python_common.md b/docs/roles/debian/python_common.md index 2c9f384f9..94573f816 100644 --- a/docs/roles/debian/python_common.md +++ b/docs/roles/debian/python_common.md @@ -9,15 +9,17 @@ Packages required by all Python scripts and applications. ```yaml --- python_common: - remove_packages: - - python-pip - - python3-yaml # linters need a newer version of PyYAML than the one that ships with Debian - - python-botocore - - python-urllib3 + cleanup: false # set to true to remove unwanted packages + remove_packages: [] # list of packages to remove if cleanup: true + # Example from ansible role + #- python-pip + #- python3-yaml # linters need a newer version of PyYAML than the one that ships with Debian + #- python-botocore + #- python-urllib3 + # Commonly required Python system packages install_packages: - python3-distutils - python3-venv - - python3-pip - cloud-init # package can get removed with python3-yaml but we need it for auto-scale ``` diff --git a/docs/roles/debian/swap.md b/docs/roles/debian/swap.md index 8bfeaa59c..ef6f5f208 100644 --- a/docs/roles/debian/swap.md +++ b/docs/roles/debian/swap.md @@ -1,6 +1,5 @@ -# General swap role to add swap space - -By default, role is set to add 2G of swap space. +# Swap +General role to add swap space to servers. By default, role is set to add 4G of swap space. As a general rule of thumb, swap should be set to 2.5 times the amount of RAM. So if a server has 2G RAM, we should set swap to 5G by setting `size: "5G"` in the variables for that server. @@ -11,7 +10,7 @@ By default, role is set to add 2G of swap space. --- swap: count: 2048 - size: "2G" + size: "4G" swap_space: swapfile ``` diff --git a/roles/debian/python_common/README.md b/roles/debian/python_common/README.md index 2c9f384f9..94573f816 100644 --- a/roles/debian/python_common/README.md +++ b/roles/debian/python_common/README.md @@ -9,15 +9,17 @@ Packages required by all Python scripts and applications. ```yaml --- python_common: - remove_packages: - - python-pip - - python3-yaml # linters need a newer version of PyYAML than the one that ships with Debian - - python-botocore - - python-urllib3 + cleanup: false # set to true to remove unwanted packages + remove_packages: [] # list of packages to remove if cleanup: true + # Example from ansible role + #- python-pip + #- python3-yaml # linters need a newer version of PyYAML than the one that ships with Debian + #- python-botocore + #- python-urllib3 + # Commonly required Python system packages install_packages: - python3-distutils - python3-venv - - python3-pip - cloud-init # package can get removed with python3-yaml but we need it for auto-scale ``` diff --git a/roles/debian/swap/README.md b/roles/debian/swap/README.md index ea66bc066..ef6f5f208 100644 --- a/roles/debian/swap/README.md +++ b/roles/debian/swap/README.md @@ -1,8 +1,5 @@ -# General swap role to add swap space - -By default, role is set to add 2G of swap space. -As a general rule of thumb, swap should be set to 2.5x of current RAM. -If server has 2G RAM, set RAM to 5G by creating swap.yml inside vars at local directory. +# Swap +General role to add swap space to servers. By default, role is set to add 4G of swap space. As a general rule of thumb, swap should be set to 2.5 times the amount of RAM. So if a server has 2G RAM, we should set swap to 5G by setting `size: "5G"` in the variables for that server. From 9b6395403bbe2bdbffa3566ba620ee97bad895e1 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Thu, 18 Jan 2024 19:39:50 +0100 Subject: [PATCH 24/33] Fixing LDAP SSL to use systemd timer. --- docs/roles/debian/ldap_server.md | 1 + roles/debian/ldap_server/README.md | 1 + roles/debian/ldap_server/defaults/main.yml | 1 + 3 files changed, 3 insertions(+) diff --git a/docs/roles/debian/ldap_server.md b/docs/roles/debian/ldap_server.md index 7fb78adfb..6514e4779 100644 --- a/docs/roles/debian/ldap_server.md +++ b/docs/roles/debian/ldap_server.md @@ -107,6 +107,7 @@ ldap_server: # reload_command: restart # reload: # - slapd + # on_calendar: "Mon *-*-* 04:00:00" replication: host: "" # host must be present in config/hosts for ce-provision, leave empty if no replication is desired port: "636" diff --git a/roles/debian/ldap_server/README.md b/roles/debian/ldap_server/README.md index 7fb78adfb..6514e4779 100644 --- a/roles/debian/ldap_server/README.md +++ b/roles/debian/ldap_server/README.md @@ -107,6 +107,7 @@ ldap_server: # reload_command: restart # reload: # - slapd + # on_calendar: "Mon *-*-* 04:00:00" replication: host: "" # host must be present in config/hosts for ce-provision, leave empty if no replication is desired port: "636" diff --git a/roles/debian/ldap_server/defaults/main.yml b/roles/debian/ldap_server/defaults/main.yml index 981d6b995..8178688a0 100644 --- a/roles/debian/ldap_server/defaults/main.yml +++ b/roles/debian/ldap_server/defaults/main.yml @@ -41,6 +41,7 @@ ldap_server: # reload_command: restart # reload: # - slapd + # on_calendar: "Mon *-*-* 04:00:00" replication: host: "" # host must be present in config/hosts for ce-provision, leave empty if no replication is desired port: "636" From 516f61d9ea0dde476856e6b216f75b1140bfac4e Mon Sep 17 00:00:00 2001 From: gregharvey Date: Thu, 18 Jan 2024 19:40:07 +0100 Subject: [PATCH 25/33] Allowing different systemd timer names for different Ansible installs. --- docs/roles/debian/ansible.md | 1 + docs/roles/debian/ce_deploy.md | 1 + docs/roles/debian/ce_provision.md | 1 + roles/_meta/common_base/meta/main.yml | 1 + roles/debian/ansible/README.md | 1 + roles/debian/ansible/defaults/main.yml | 1 + roles/debian/ansible/tasks/main.yml | 9 +++++---- roles/debian/ce_deploy/README.md | 1 + roles/debian/ce_deploy/defaults/main.yml | 1 + roles/debian/ce_deploy/tasks/main.yml | 1 + roles/debian/ce_provision/README.md | 1 + roles/debian/ce_provision/defaults/main.yml | 1 + roles/debian/ce_provision/tasks/main.yml | 1 + 13 files changed, 17 insertions(+), 4 deletions(-) diff --git a/docs/roles/debian/ansible.md b/docs/roles/debian/ansible.md index b7dfe0dbc..f50de4929 100644 --- a/docs/roles/debian/ansible.md +++ b/docs/roles/debian/ansible.md @@ -19,6 +19,7 @@ ce_ansible: enabled: true # create systemd timer to auto-upgrade Ansible command: "{{ _venv_path }}/bin/python3 -m pip install --upgrade ansible" # if you set venv_path above then set it here too on_calendar: "*-*-* 01:30:00" # see systemd.time documentation - https://www.freedesktop.org/software/systemd/man/latest/systemd.time.html#Calendar%20Events + #timer_name: upgrade_ansible linters: enabled: true # will not install linters if false, installing linters breaks cloud-init diff --git a/docs/roles/debian/ce_deploy.md b/docs/roles/debian/ce_deploy.md index d020663ab..f9e55b15f 100644 --- a/docs/roles/debian/ce_deploy.md +++ b/docs/roles/debian/ce_deploy.md @@ -15,6 +15,7 @@ ce_deploy: venv_path: "/home/{{ _ce_deploy.username }}/ansible" venv_command: /usr/bin/python3 -m venv install_username: "{{ _ce_deploy.username }}" + upgrade_timer_name: upgrade_ce_deploy_ansible # Other ce-deploy settings. new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user key_name: id_rsa.pub # existing users may have a key of a different name diff --git a/docs/roles/debian/ce_provision.md b/docs/roles/debian/ce_provision.md index e8f83ea13..dba324120 100644 --- a/docs/roles/debian/ce_provision.md +++ b/docs/roles/debian/ce_provision.md @@ -16,6 +16,7 @@ ce_provision: venv_path: "/home/{{ _ce_provision.username }}/ansible" venv_command: /usr/bin/python3 -m venv install_username: "{{ _ce_provision.username }}" + upgrade_timer_name: upgrade_ce_provision_ansible # Other ce-provision settings. username: "{{ _ce_provision.username }}" new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user diff --git a/roles/_meta/common_base/meta/main.yml b/roles/_meta/common_base/meta/main.yml index 9e3635fa6..7a79993e8 100644 --- a/roles/_meta/common_base/meta/main.yml +++ b/roles/_meta/common_base/meta/main.yml @@ -51,5 +51,6 @@ dependencies: enabled: true command: "/home/{{ ce_provision.username }}/ansible/bin/python3 -m pip install --upgrade ansible" on_calendar: "*-*-* 01:30:00" + timer_name: upgrade_ansible linters: enabled: false diff --git a/roles/debian/ansible/README.md b/roles/debian/ansible/README.md index b7dfe0dbc..f50de4929 100644 --- a/roles/debian/ansible/README.md +++ b/roles/debian/ansible/README.md @@ -19,6 +19,7 @@ ce_ansible: enabled: true # create systemd timer to auto-upgrade Ansible command: "{{ _venv_path }}/bin/python3 -m pip install --upgrade ansible" # if you set venv_path above then set it here too on_calendar: "*-*-* 01:30:00" # see systemd.time documentation - https://www.freedesktop.org/software/systemd/man/latest/systemd.time.html#Calendar%20Events + #timer_name: upgrade_ansible linters: enabled: true # will not install linters if false, installing linters breaks cloud-init diff --git a/roles/debian/ansible/defaults/main.yml b/roles/debian/ansible/defaults/main.yml index f58a026b0..949bb48b7 100644 --- a/roles/debian/ansible/defaults/main.yml +++ b/roles/debian/ansible/defaults/main.yml @@ -8,5 +8,6 @@ ce_ansible: enabled: true # create systemd timer to auto-upgrade Ansible command: "{{ _venv_path }}/bin/python3 -m pip install --upgrade ansible" # if you set venv_path above then set it here too on_calendar: "*-*-* 01:30:00" # see systemd.time documentation - https://www.freedesktop.org/software/systemd/man/latest/systemd.time.html#Calendar%20Events + #timer_name: upgrade_ansible linters: enabled: true # will not install linters if false, installing linters breaks cloud-init diff --git a/roles/debian/ansible/tasks/main.yml b/roles/debian/ansible/tasks/main.yml index 56d1d94f2..b1aa47a7f 100644 --- a/roles/debian/ansible/tasks/main.yml +++ b/roles/debian/ansible/tasks/main.yml @@ -68,12 +68,13 @@ content: "export PATH=$PATH:{{ ce_ansible.venv_path | default(_venv_path) }}/bin" dest: "/etc/profile.d/ansible-path.sh" +- name: Build systemd timer variables. + ansible.builtin.set_fact: + _timer: {"{{ ce_ansible.upgrade.timer_name | default(_ce_ansible_timer_name) }}": { timer_command: "{{ ce_ansible.upgrade.command }}", timer_OnCalendar: "{{ ce_ansible.upgrade.on_calendar }}" }} + - name: Create systemd timer to upgrade Ansible. ansible.builtin.include_role: name: contrib/systemd_timers vars: - timers: - upgrade_ansible: - timer_command: "{{ ce_ansible.upgrade.command }}" - timer_OnCalendar: "{{ ce_ansible.upgrade.on_calendar }}" + timers: "{{ _timer }}" when: ce_ansible.upgrade.enabled diff --git a/roles/debian/ce_deploy/README.md b/roles/debian/ce_deploy/README.md index d020663ab..f9e55b15f 100644 --- a/roles/debian/ce_deploy/README.md +++ b/roles/debian/ce_deploy/README.md @@ -15,6 +15,7 @@ ce_deploy: venv_path: "/home/{{ _ce_deploy.username }}/ansible" venv_command: /usr/bin/python3 -m venv install_username: "{{ _ce_deploy.username }}" + upgrade_timer_name: upgrade_ce_deploy_ansible # Other ce-deploy settings. new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user key_name: id_rsa.pub # existing users may have a key of a different name diff --git a/roles/debian/ce_deploy/defaults/main.yml b/roles/debian/ce_deploy/defaults/main.yml index 52bd86415..1dc9ba5b3 100644 --- a/roles/debian/ce_deploy/defaults/main.yml +++ b/roles/debian/ce_deploy/defaults/main.yml @@ -7,6 +7,7 @@ ce_deploy: venv_path: "/home/{{ _ce_deploy.username }}/ansible" venv_command: /usr/bin/python3 -m venv install_username: "{{ _ce_deploy.username }}" + upgrade_timer_name: upgrade_ce_deploy_ansible # Other ce-deploy settings. new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user key_name: id_rsa.pub # existing users may have a key of a different name diff --git a/roles/debian/ce_deploy/tasks/main.yml b/roles/debian/ce_deploy/tasks/main.yml index a9d607723..ae5395780 100644 --- a/roles/debian/ce_deploy/tasks/main.yml +++ b/roles/debian/ce_deploy/tasks/main.yml @@ -133,6 +133,7 @@ _venv_path: "{{ ce_deploy.venv_path }}" _venv_command: "{{ ce_deploy.venv_command }}" _install_username: "{{ ce_deploy.install_username }}" + _ce_ansible_timer_name: "{{ ce_deploy.upgrade_timer_name }}" - name: Install Ansible. ansible.builtin.include_role: diff --git a/roles/debian/ce_provision/README.md b/roles/debian/ce_provision/README.md index e8f83ea13..dba324120 100644 --- a/roles/debian/ce_provision/README.md +++ b/roles/debian/ce_provision/README.md @@ -16,6 +16,7 @@ ce_provision: venv_path: "/home/{{ _ce_provision.username }}/ansible" venv_command: /usr/bin/python3 -m venv install_username: "{{ _ce_provision.username }}" + upgrade_timer_name: upgrade_ce_provision_ansible # Other ce-provision settings. username: "{{ _ce_provision.username }}" new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user diff --git a/roles/debian/ce_provision/defaults/main.yml b/roles/debian/ce_provision/defaults/main.yml index 4c266e3e0..8f8d508cf 100644 --- a/roles/debian/ce_provision/defaults/main.yml +++ b/roles/debian/ce_provision/defaults/main.yml @@ -8,6 +8,7 @@ ce_provision: venv_path: "/home/{{ _ce_provision.username }}/ansible" venv_command: /usr/bin/python3 -m venv install_username: "{{ _ce_provision.username }}" + upgrade_timer_name: upgrade_ce_provision_ansible # Other ce-provision settings. username: "{{ _ce_provision.username }}" new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user diff --git a/roles/debian/ce_provision/tasks/main.yml b/roles/debian/ce_provision/tasks/main.yml index ca59d5928..0f4edeab1 100644 --- a/roles/debian/ce_provision/tasks/main.yml +++ b/roles/debian/ce_provision/tasks/main.yml @@ -141,6 +141,7 @@ _venv_path: "{{ ce_provision.venv_path }}" _venv_command: "{{ ce_provision.venv_command }}" _install_username: "{{ ce_provision.install_username }}" + _ce_ansible_timer_name: "{{ ce_provision.upgrade_timer_name }}" - name: Install Ansible. ansible.builtin.include_role: From 17f7804c427e1b3785b30a72c2f9a1762f3ebae9 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Fri, 19 Jan 2024 10:29:40 +0100 Subject: [PATCH 26/33] Fixing dynamic key name in ansible role. --- roles/debian/ansible/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/debian/ansible/tasks/main.yml b/roles/debian/ansible/tasks/main.yml index b1aa47a7f..8cbdc9241 100644 --- a/roles/debian/ansible/tasks/main.yml +++ b/roles/debian/ansible/tasks/main.yml @@ -70,7 +70,7 @@ - name: Build systemd timer variables. ansible.builtin.set_fact: - _timer: {"{{ ce_ansible.upgrade.timer_name | default(_ce_ansible_timer_name) }}": { timer_command: "{{ ce_ansible.upgrade.command }}", timer_OnCalendar: "{{ ce_ansible.upgrade.on_calendar }}" }} + _timer: "{'{{ ce_ansible.upgrade.timer_name | default(_ce_ansible_timer_name) }}': { timer_command: '{{ ce_ansible.upgrade.command }}', timer_OnCalendar: '{{ ce_ansible.upgrade.on_calendar }}' }}" - name: Create systemd timer to upgrade Ansible. ansible.builtin.include_role: From 3f9f67e2ecb40cabb45fdabd7f3be5af5cdbe15a Mon Sep 17 00:00:00 2001 From: gregharvey Date: Fri, 19 Jan 2024 10:49:26 +0100 Subject: [PATCH 27/33] Trying to debug missing timer_command var. --- roles/debian/ansible/tasks/main.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/roles/debian/ansible/tasks/main.yml b/roles/debian/ansible/tasks/main.yml index 8cbdc9241..5ed23549e 100644 --- a/roles/debian/ansible/tasks/main.yml +++ b/roles/debian/ansible/tasks/main.yml @@ -72,6 +72,9 @@ ansible.builtin.set_fact: _timer: "{'{{ ce_ansible.upgrade.timer_name | default(_ce_ansible_timer_name) }}': { timer_command: '{{ ce_ansible.upgrade.command }}', timer_OnCalendar: '{{ ce_ansible.upgrade.on_calendar }}' }}" +- debug: + msg: "{{ _timer }}" + - name: Create systemd timer to upgrade Ansible. ansible.builtin.include_role: name: contrib/systemd_timers From e34d247f22be1540e08717b05131def007bcf776 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Fri, 19 Jan 2024 11:07:08 +0100 Subject: [PATCH 28/33] Treating the timer string so it becomes a dict. --- roles/debian/ansible/tasks/main.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/roles/debian/ansible/tasks/main.yml b/roles/debian/ansible/tasks/main.yml index 5ed23549e..a34b6d0fe 100644 --- a/roles/debian/ansible/tasks/main.yml +++ b/roles/debian/ansible/tasks/main.yml @@ -68,12 +68,13 @@ content: "export PATH=$PATH:{{ ce_ansible.venv_path | default(_venv_path) }}/bin" dest: "/etc/profile.d/ansible-path.sh" -- name: Build systemd timer variables. +- name: Build systemd timer variables string. ansible.builtin.set_fact: _timer: "{'{{ ce_ansible.upgrade.timer_name | default(_ce_ansible_timer_name) }}': { timer_command: '{{ ce_ansible.upgrade.command }}', timer_OnCalendar: '{{ ce_ansible.upgrade.on_calendar }}' }}" -- debug: - msg: "{{ _timer }}" +- name: Turn the timer string into a dictionary. + ansible.builtin.set_fact: + _timer: "{{ _timer.splitlines()|map('from_yaml')|list }}" - name: Create systemd timer to upgrade Ansible. ansible.builtin.include_role: From 170c782caf1f483f28bacd8f5f5184cacf16a831 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Mon, 22 Jan 2024 12:22:45 +0100 Subject: [PATCH 29/33] Moving default log location for clamav. --- roles/debian/clamav/defaults/main.yml | 3 ++- roles/debian/clamav/templates/clamscan.j2 | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/roles/debian/clamav/defaults/main.yml b/roles/debian/clamav/defaults/main.yml index 651cafde3..6185d71fa 100644 --- a/roles/debian/clamav/defaults/main.yml +++ b/roles/debian/clamav/defaults/main.yml @@ -4,7 +4,8 @@ clamav: server_name: "{{ inventory_hostname }}" # for identification via email, defaults to Ansible inventory name. on_calendar: "*-*-* 02:30:00" # see systemd.time documentation - https://www.freedesktop.org/software/systemd/man/latest/systemd.time.html#Calendar%20Events scripts_location: /usr/local/clamav/script - log_location: /usr/local/clamav/log + log_location: /var/log/clamav + log_name: clamav.log send_mail: false # Important: will not send any emails by default. send_on_fail: true # Only sends emails on scan failure, will not email for successful scans. report_recipient_email: mail@example.com diff --git a/roles/debian/clamav/templates/clamscan.j2 b/roles/debian/clamav/templates/clamscan.j2 index 0142a1b8b..c2b54e2cb 100644 --- a/roles/debian/clamav/templates/clamscan.j2 +++ b/roles/debian/clamav/templates/clamscan.j2 @@ -1,7 +1,7 @@ #!/bin/bash SUBJECT="{{ clamav.server_name }} PASSED SCAN" EMAIL="{{ clamav.report_recipient_email }}" -LOG={{ clamav.log_location }}/clamav.log +LOG={{ clamav.log_location }}/{{ clamav.log_name }} TMP_LOG=/tmp/clam.daily rm -rf ${TMP_LOG} From 21f882b8376ee1af0f77247ed9121648c62e7e6d Mon Sep 17 00:00:00 2001 From: gregharvey Date: Mon, 22 Jan 2024 12:23:47 +0100 Subject: [PATCH 30/33] Updating ClamAV docs. --- docs/roles/debian/clamav.md | 3 ++- roles/debian/clamav/README.md | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/roles/debian/clamav.md b/docs/roles/debian/clamav.md index 81a888707..8ce643b7e 100644 --- a/docs/roles/debian/clamav.md +++ b/docs/roles/debian/clamav.md @@ -15,7 +15,8 @@ clamav: server_name: "{{ inventory_hostname }}" # for identification via email, defaults to Ansible inventory name. on_calendar: "*-*-* 02:30:00" # see systemd.time documentation - https://www.freedesktop.org/software/systemd/man/latest/systemd.time.html#Calendar%20Events scripts_location: /usr/local/clamav/script - log_location: /usr/local/clamav/log + log_location: /var/log/clamav + log_name: clamav.log send_mail: false # Important: will not send any emails by default. send_on_fail: true # Only sends emails on scan failure, will not email for successful scans. report_recipient_email: mail@example.com diff --git a/roles/debian/clamav/README.md b/roles/debian/clamav/README.md index 81a888707..8ce643b7e 100644 --- a/roles/debian/clamav/README.md +++ b/roles/debian/clamav/README.md @@ -15,7 +15,8 @@ clamav: server_name: "{{ inventory_hostname }}" # for identification via email, defaults to Ansible inventory name. on_calendar: "*-*-* 02:30:00" # see systemd.time documentation - https://www.freedesktop.org/software/systemd/man/latest/systemd.time.html#Calendar%20Events scripts_location: /usr/local/clamav/script - log_location: /usr/local/clamav/log + log_location: /var/log/clamav + log_name: clamav.log send_mail: false # Important: will not send any emails by default. send_on_fail: true # Only sends emails on scan failure, will not email for successful scans. report_recipient_email: mail@example.com From a57c5ffa6862ec3daa56e3cca95ce639d7ee9b4e Mon Sep 17 00:00:00 2001 From: gregharvey Date: Thu, 25 Jan 2024 13:32:37 +0100 Subject: [PATCH 31/33] Grouping systemd timer tasks together. --- roles/debian/ansible/tasks/main.yml | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/roles/debian/ansible/tasks/main.yml b/roles/debian/ansible/tasks/main.yml index b845f2567..f14197d28 100644 --- a/roles/debian/ansible/tasks/main.yml +++ b/roles/debian/ansible/tasks/main.yml @@ -68,17 +68,19 @@ content: "export PATH=$PATH:{{ ce_ansible.venv_path | default(_venv_path) }}/bin" dest: "/etc/profile.d/ansible-path.sh" -- name: Build systemd timer variables string. - ansible.builtin.set_fact: - _timer: "{'{{ ce_ansible.upgrade.timer_name | default(_ce_ansible_timer_name) }}': { timer_command: '{{ ce_ansible.upgrade.command }}', timer_OnCalendar: '{{ ce_ansible.upgrade.on_calendar }}' }}" +- name: Install systemd timer. + when: ce_ansible.upgrade.enabled + block: + - name: Build systemd timer variables string. + ansible.builtin.set_fact: + _timer: "{'{{ ce_ansible.upgrade.timer_name | default(_ce_ansible_timer_name) }}': { timer_command: '{{ ce_ansible.upgrade.command }}', timer_OnCalendar: '{{ ce_ansible.upgrade.on_calendar }}' }}" -- name: Turn the timer string into a dictionary. - ansible.builtin.set_fact: - _timer: "{{ _timer.splitlines()|map('from_yaml')|list }}" + - name: Turn the timer string into a dictionary. + ansible.builtin.set_fact: + _timer: "{{ _timer.splitlines()|map('from_yaml')|list }}" -- name: Create systemd timer to upgrade Ansible. - ansible.builtin.include_role: - name: contrib/systemd_timers - vars: - timers: "{{ _timer }}" - when: ce_ansible.upgrade.enabled + - name: Create systemd timer to upgrade Ansible. + ansible.builtin.include_role: + name: contrib/systemd_timers + vars: + timers: "{{ _timer }}" From 36a52eeb2cbea9e18d66a43b1d88e692c8dfd842 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Wed, 21 Feb 2024 16:47:33 +0100 Subject: [PATCH 32/33] Exposing ce-provision version in build output. --- roles/_init/defaults/main.yml | 1 + roles/_init/tasks/main.yml | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/roles/_init/defaults/main.yml b/roles/_init/defaults/main.yml index 01249906a..7b7c06560 100644 --- a/roles/_init/defaults/main.yml +++ b/roles/_init/defaults/main.yml @@ -4,6 +4,7 @@ _init: # This is used to detect if the playbook must re-run or not. vars_dirs: [] force_play: false + ce_provision_version: 2.x # Outputted by the _init role at the start of plays. # Although these variables logically belong with ce_provision, the _init role needs to # gather the extra variables if there are any, so there are _init variables. diff --git a/roles/_init/tasks/main.yml b/roles/_init/tasks/main.yml index 39c368378..357f94c5e 100644 --- a/roles/_init/tasks/main.yml +++ b/roles/_init/tasks/main.yml @@ -1,4 +1,8 @@ --- +- name: Version check. + ansible.builtin.debug: + msg: "Using ce-provision {{ _init.ce_provision_version }}" + - name: Check OS family. when: ansible_os_family != "Debian" block: From a02e9e5fbade7ddf56ec3d8d73a3df062fb08e84 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Fri, 23 Feb 2024 18:24:49 +0100 Subject: [PATCH 33/33] Wrong variable in meta role for controller username. --- roles/_meta/common_base/meta/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/_meta/common_base/meta/main.yml b/roles/_meta/common_base/meta/main.yml index 7a79993e8..70c522c53 100644 --- a/roles/_meta/common_base/meta/main.yml +++ b/roles/_meta/common_base/meta/main.yml @@ -44,12 +44,12 @@ dependencies: when: is_local is not defined or not is_local - role: debian/ansible ce_ansible: - venv_path: "/home/{{ ce_provision.username }}/ansible" + venv_path: "/home/{{ user_provision.username }}/ansible" venv_command: "/usr/bin/python3 -m venv" - install_username: "{{ ce_provision.username }}" + install_username: "{{ user_provision.username }}" upgrade: enabled: true - command: "/home/{{ ce_provision.username }}/ansible/bin/python3 -m pip install --upgrade ansible" + command: "/home/{{ user_provision.username }}/ansible/bin/python3 -m pip install --upgrade ansible" on_calendar: "*-*-* 01:30:00" timer_name: upgrade_ansible linters: