update ansible installation files

README.md

@@ -158,6 +158,7 @@ Thanks go to these people!
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/arvid123"><img src="https://github.com/arvid123.png" width="100px;" alt="Arvid Lagerqvist"/><br /><sub><b>Arvid Lagerqvist</b></sub></a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/nforsg"><img src="https://github.com/nforsg.png" width="100px;" alt="Nils Forsgren"/><br /><sub><b>Nils Forsgren</b></sub></a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/foroughsh"><img src="https://github.com/foroughsh.png" width="100px;" alt="Forough Shahab Samani"/><br /><sub><b>Forough Shahab Samani</b></sub></a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/ppplbngth"><img src="https://github.com/ppplbngth.png" width="100px;" alt="Benth Roland Pappila"/><br /><sub><b>Benth Roland Pappila</b></sub></a></td>
     </tr>
   </tbody>
 </table>

ansible/README.md

@@ -13,21 +13,43 @@ Ansible can be installed by running the command:
 pip install ansible
 ```
 
+## Setup SSH keys
+
+Before starting the installation, make sure to setup SSH key pairs on all of the servers where CSLE will be installed
+and make sure that you can reach all of those servers from the workstation/laptop where the ansible playbooks will be run.
+To generate an ssh key pair on a server, run the command `ssh-keygen`. Once keys have been created on all hosts, 
+copy the public key (e.g., `~/.ssh/id_rsa.pub`) of each host to the file `.ssh/authorized_keys` of the other hosts.
+After creating the SSH keys and verifying the connections, define the path to the private key of the host from which
+the installation will run in the file [ansible.cfg](ansible.cfg).
+
 ## Configuring the installation
-Before starting the CSLE installation, open the file XXX and configure the following variables:
 
-- user
-- todo
+Before starting the CSLE installation, do the following.
+
+1. Specify the leader and the worker hosts in [inventory](inventory)
+2. Configure installation variables (e.g., the user of the installation and the Python version) in [group_vars/all/variables.yml](group_vars/all/variables.yml). 
+
 
 ## Installing CSLE
-To install CSLE with ansible, run the following commands:
+To install CSLE with ansible, run the following playbook:
+
 ```bash
 ansible-playbook --ask-become-pass installing_the_management_system.yml
 ```
+Alternatively, you can run each playbook manually by executing the commands:
+```bash
+ansible-playbook --ask-become-pass install_setup.yml
+ansible-playbook --ask-become-pass install_metastore.yml
+ansible-playbook --ask-become-pass install_simulation_system.yml
+ansible-playbook --ask-become-pass install_emulation_system.yml
+ansible-playbook --ask-become-pass install_management_system.yml
+```
 
 ### Debugging
-If the installation fails at some step, you can debug the reason for the failure by adding the following 
-line to the Ansible playbook. First, we register a variable that holds a dictionary of the output for the module in that task. In the given example git_installation is this variable. In the next lines, we use debug to print the variable. 
+
+If the installation fails at some step, you can debug the reason for the failure as follows.  
+Start by registering a variable that holds a dictionary of the output for the module in the task that failed (`git_installation` in the example below). 
+After registering the variable, add it to the debug group. 
 
 ```bash
 - name: Installation of git

ansible/group_vars/all/variables.yml

@@ -0,0 +1,12 @@
+user: "shahab"
+csle_git_repo_url: "https://github.com/Limmen/csle"
+leader_ip: "172.31.212.83"
+nginx_content_name: "nginx_configuration_content.txt"
+web_port: 7777
+postgres_version: "15"
+postgres_citus_version: "postgresql-15-citus-11.2"
+postgresql_url: "https://install.citusdata.com/community/deb.sh"
+anaconda_url: "https://repo.anaconda.com/archive/Anaconda3-2022.10-Linux-x86_64.sh"
+anaconda_install_script: "Anaconda3-2022.10-Linux-x86_64.sh"
+conda_environment_name: "base"
+python_version: "3.9"

ansible/install.yml

@@ -0,0 +1,21 @@
+---
+
+- hosts: all
+  become: true
+
+  tasks:
+
+    - name: Installation setup
+      ansible.builtin.import_playbook: install_setup.yaml
+
+    - name: Install metastore
+        ansible.builtin.import_playbook: install_metastore.yaml
+
+    - name: Install simulation system
+        ansible.builtin.import_playbook: install_simulation_system.yaml
+
+    - name: Install emulation system
+        ansible.builtin.import_playbook: install_emulation_system.yaml
+
+    - name: Install management system
+        ansible.builtin.import_playbook: install_management_system.yaml

ansible/installing_emulation_system.yml → ansible/install_emulation_system.yml

@@ -3,18 +3,14 @@
 - hosts: all
   become: true
 
-  vars:
-    user: "shahab"
-    leader_ip: "172.31.212.83"
-
   tasks:
 
-  - name: Check if keyrings folder exist
+  - name: Check if apt keyrings directory exist
     stat:
       path: /etc/apt/keyrings
     register: keyrings_directory_exists
 
-  - name: Creates keyrings directory if it does not exist
+  - name: Create keyrings directory if it does not exist
     ansible.builtin.file:
       path: /etc/apt/keyrings
       state: directory
@@ -30,7 +26,7 @@
           interact
         '
         
-  - name: Second command for installing keys
+  - name: Install docker apt keys
     shell: echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
 
   - name: Update package cache
@@ -51,25 +47,25 @@
       state: present
 
   - name: Add user to Docker group
-    shell: sudo usermod -aG docker $USER
+    shell: "sudo usermod -aG docker {{ user }}"
 
   - name: Initialize Docker Swarm
     command: docker swarm init --advertise-addr "{{ leader_ip }}"
     ignore_errors: yes
 
-  - name: Pulling base image
+  - name: Pulling base Docker images
     shell: "cd /home/{{ user }}/csle/emulation-system/base_images && make pull"
 
-  - name: Pulling derived image
+  - name: Pulling derived Docker images
     shell: |
       cd "/home/{{ user }}/csle/emulation-system/derived_images" && \
       make pull
     args:
       executable: /bin/bash
 
-  - name: Installing the envs on the leader node
+  - name: Installing the emulation environments on the leader node
     shell: |
-      source "/home/{{ user }}/anaconda3/bin/activate" py39 && \
+      source "/home/{{ user }}/anaconda3/bin/activate {{ conda_environment_name }}" && \
       cd "/home/{{ user }}/csle/emulation-system/envs" && \
       make install
     args:

ansible/installing_the_management_system.yml → ansible/install_management_system.yml

@@ -1,18 +1,10 @@
 ---
 
 - hosts: all
-  #become: true
-
-  vars:
-    user: "shahab"
-    leader_ip: "172.31.212.83"
-    nginx_content_name: "Nginx_configuration_content.txt"
-    web_port: 7777
 
 
   tasks:
 
-
   - name: Check if nvm is installed
     shell: |
       source "/home/{{ user }}/.nvm/nvm.sh" && \
@@ -119,7 +111,7 @@
     ignore_errors: true
     register: port_check_result
 
-  - name: install and start pgadmin
+  - name: Install and start pgadmin
     become: true
     shell: |
       docker pull dpage/pgadmin4
@@ -196,7 +188,7 @@
     when: variable_exists.rc != 0
 
   - name: CSLE init
-    shell: "source /home/{{ user }}/anaconda3/bin/activate py39 && echo $CSLE_HOME && csle init"
+    shell: "source /home/{{ user }}/anaconda3/bin/activate {{ conda_environment_name }} && echo $CSLE_HOME && csle init"
     args:
       executable: /bin/bash
     environment: 

ansible/installing_the_metastore.yml → ansible/install_metastore.yml

@@ -3,13 +3,6 @@
 - hosts: all
   become: true
 
-  vars:
-    csle_git_repo_url: "https://github.com/Limmen/csle"
-    postgresql_url: "https://install.citusdata.com/community/deb.sh"
-    user: "shahab"
-    postgres_version: "15"
-    leader_IP: "172.31.212.90"
-
   tasks:
 
     - name: Check if PostgreSQL is installed, if you see error here just ignore it!
@@ -20,7 +13,7 @@
     - name: Download PostgreSQL
       ansible.builtin.get_url:
         url: "{{ postgresql_url }}"
-        dest: /home/shahab
+        dest: "/home/{{ user }}"
         mode: '0770'
       when: postgresql_installed.rc != 0
 
@@ -30,7 +23,7 @@
 
     - name: Install PostgreSQL Citus extension
       apt:
-        name: postgresql-15-citus-11.2
+        name: "{{ postgres_citus_version }}"
         state: present
         update_cache: yes
       when: postgresql_installed.rc != 0
@@ -58,10 +51,10 @@
         state: present
 
     - name: Set shared_preload_libraries using pg_conftool
-      shell: "pg_conftool 15 main set shared_preload_libraries citus"
+      shell: "pg_conftool {{ postgres_version }} main set shared_preload_libraries citus"
 
     - name: Set listen address using pg_conftool
-      shell: "pg_conftool 15 main set listen_addresses '*'"
+      shell: "pg_conftool {{ postgres_version }} main set listen_addresses '*'"
 
 
     - name: Run psql and set password for postgres user
@@ -123,7 +116,7 @@
       lineinfile:
         path: "/home/{{ user }}/csle/metastore/create_cluster.sql"
         regexp: '^SELECT citus_set_coordinator_host'
-        line: "SELECT citus_set_coordinator_host('{{ leader_IP }}', 5432);"
+        line: "SELECT citus_set_coordinator_host('{{ leader_ip }}', 5432);"
 
     - name: Create PostgreSQL cluster
       expect:
@@ -149,5 +142,5 @@
       file:
         path: /var/log/postgresql
         owner: "{{ user }}"
-        mode: "0775"  # Specify the desired permissions here
+        mode: "0775"
       become: true

ansible/installation_setup.yml → ansible/install_setup.yml

@@ -3,11 +3,6 @@
 - hosts: all
   become: true
 
-  vars:
-    csle_git_repo_url: "https://github.com/Limmen/csle"
-    anaconda_url: "https://repo.anaconda.com/archive/Anaconda3-2022.10-Linux-x86_64.sh"
-    user: "shahab"
-
   tasks:
 
   - name: Installation of build-essential
@@ -35,22 +30,22 @@
     register: anaconda_installed
     ignore_errors: true
 
-  - name: Download anaconda
+  - name: Download Anaconda
     ansible.builtin.get_url:
       url: "{{ anaconda_url }}"
       dest: "/home/{{ user }}"
       mode: '0770'
     when: anaconda_installed.rc != 0
 
-  - name: Install anaconda
-    shell: "/home/{{ user }}/Anaconda3-2022.10-Linux-x86_64.sh -b -u -p /home/{{ user }}/anaconda3"
+  - name: Install Anaconda
+    shell: "/home/{{ user }}/{{ anaconda_install_script }} -b -u -p /home/{{ user }}/anaconda3"
     when: anaconda_installed.rc != 0
 
-  - name: Add anaconda bin to path
+  - name: Add Anaconda bin to path
     shell: "echo export PATH=/home/{{ user }}/anaconda3/bin:$PATH >> /etc/profile"
     when: anaconda_installed.rc != 0
 
-  - name: Conda - read permission for all
+  - name: Set read permission for all on the Anaconda folder
     file:
       path: "/home/{{ user }}/anaconda3"
       mode: +r
@@ -93,7 +88,7 @@
       path: /var/log/csle
     register: log_directory_exists
 
-  - name: Creates CSLE log directory
+  - name: Create the CSLE log directory
     ansible.builtin.file:
       path: /var/log/csle
       state: directory 
@@ -106,15 +101,15 @@
       path: /tmp/csle
     register: tmp_directory_exists
 
-  - name: Creates CSLE tmp directory
+  - name: Create the CSLE tmp directory
     ansible.builtin.file:
       path: /tmp/csle
       state: directory
       mode: "0774"
       owner: "{{ user }}"
     when: not tmp_directory_exists.stat.exists
 
-  - name: Add or modify sudoers configuration
+  - name: Add or modify the sudoers configuration
     lineinfile:
       path: /etc/sudoers
       line: "{{ user }} ALL= NOPASSWD: /usr/sbin/service docker stop, /usr/sbin/service docker start, /usr/sbin/service docker restart, /usr/sbin/service nginx stop, /usr/sbin/service nginx start, /usr/sbin/service nginx restart, /usr/sbin/service postgresql start, /usr/sbin/service postgresql stop, /usr/sbin/service postgresql restart, /bin/kill, /usr/bin/journalctl -u docker.service -n 100 --no-pager -e"

ansible/install_simulation_system.yml

@@ -0,0 +1,51 @@
+---
+
+- hosts: all
+  become: true
+
+  tasks:
+
+  - name: List all Conda environments
+    shell: "/home/{{ user }}/anaconda3/bin/conda env list"
+    register: conda_env_list
+    changed_when: false
+
+  - name: Check if the conda environment exists
+    set_fact:
+      conda_env_exists: "{{ '{{ conda_environment_name }}' in conda_env_list.stdout }}"
+
+  - name: Create and activate the conda environment if it does not exist
+    shell: |
+      "/home/{{ user }}/anaconda3/bin/conda create -y -n {{ conda_environment_name }} python={{ python_version }}" && \
+      source "/home/{{ user }}/anaconda3/bin/activate {{ conda_environment_name }}" && \
+      "/home/{{ user }}/anaconda3/bin/conda" install -y pip
+    args:
+      executable: /bin/bash
+    when: not conda_env_exists
+
+  - name: Add execute permissions on Python install script
+      file:
+        path: "/home/{{ user }}/csle/simulation-system/libs/local_install.sh"
+        mode: +rwx
+        recurse: yes
+
+  - name: Add execute permissions on Python dev-install script
+      file:
+        path: "/home/{{ user }}/csle/simulation-system/libs/local_install_dev.sh"
+        mode: +rwx
+        recurse: yes
+
+  - name: Activate the environment if it exist and install CSLE Python libraries
+    shell: |
+      source "/home/{{ user }}/anaconda3/bin/activate {{ conda_environment_name }}" && \
+      cd "/home/{{ user }}/csle/simulation-system/libs" && \
+      ./local_install.sh && \
+      ./local_install_dev.sh
+     
+  - name: Install CSLE simulation environments on the leader node
+    shell: |
+      source "/home/{{ user }}/anaconda3/bin/activate {{ conda_environment_name }}" && \
+      cd "/home/{{ user }}/csle/simulation-system/envs" && \
+      make install
+    args:
+      executable: /bin/bash