From a5ec2c05af65ca076a61b981ace57cb39dacd25e Mon Sep 17 00:00:00 2001 From: Antonio Fresneda Date: Fri, 23 Jul 2021 12:55:52 +0200 Subject: [PATCH 01/10] Add vagrant wrappers --- deps/deployment/VagrantWrapper.py | 107 +++++++++++++++++++++++ deps/deployment/Vagrantfile.py | 73 ++++++++++++++++ deps/deployment/vagrantfile_template.txt | 24 +++++ 3 files changed, 204 insertions(+) create mode 100644 deps/deployment/VagrantWrapper.py create mode 100644 deps/deployment/Vagrantfile.py create mode 100644 deps/deployment/vagrantfile_template.txt diff --git a/deps/deployment/VagrantWrapper.py b/deps/deployment/VagrantWrapper.py new file mode 100644 index 0000000000..08cf025143 --- /dev/null +++ b/deps/deployment/VagrantWrapper.py @@ -0,0 +1,107 @@ +# Copyright (C) 2015-2021, Wazuh Inc. +# Created by Wazuh, Inc. . +# This program is free software; you can redistribute it and/or modify it under the terms of GPLv2 +import Vagrantfile as vfile +import vagrant +import os +from Instance import Instance + +class VagrantWrapper(Instance): + """Class to handle Vagrant operations. The class will use the Vagrantfile class to create a vagrantfile in + runtime. The vagrantfile will be dumped to disk only if the up method is executed. + + Attributes: + vagrantfile (Vagrantfile): Vagrantfile object containing the vagrantfile information. + vagrant (Vagrant): Vagrant object to handle vagrant operations + Args: + vagrant_root_folder (str): Root folder where the vagrant environment will be created. + vm_box (str): Name or link to the Vagrant box + vm_name (str): Name that will be assigned to the VM + vm_label (str): Label used in the vagrantfile. + vm_cpus (int): Number of CPUs assigned to the VM. + vm_memory (int): Number of RAM bytes assigned to the VM. + vm_system (str): System of the VM (Linux, Windows, Solaris, etc) + vm_ip (str): IP assigned to the VM. + quiet_out (Boolean): Flag to ignore the vagrant output. Defaults to True. + """ + def __init__(self, vagrant_root_folder, vm_box, vm_label, vm_name, vm_cpus, vm_memory, vm_system, vm_ip, + quiet_out=True): + + box_folder = os.path.join(vagrant_root_folder, vm_name) + os.makedirs(box_folder, exist_ok=True) + + self.vagrantfile = vfile.Vagrantfile(box_folder, vm_box, vm_label, vm_name, vm_cpus, vm_memory, + vm_system, vm_ip) + + self.vagrant = vagrant.Vagrant(root=box_folder, quiet_stdout=quiet_out, quiet_stderr=False) + self.vagrantfile.write_vagrantfile() + + + def run(self): + """Writes the vagrantfile and starts the VM specified in the vagrantfile.""" + self.vagrant.up() + + + def halt(self): + """Stops the VM specified in the vagrantfile.""" + self.vagrant.halt() + + + def restart(self): + """Restarts the VM specified in the vagrantfile.""" + self.vagrant.restart() + + + def destroy(self): + """Destroys the VM specified in the vagrantfile and remove the vagrantfile.""" + self.vagrant.destroy() + self.vagrantfile.remove_vagrantfile() + + def suspend(self): + """Suspends the VM specified in the vagrantfile.""" + self.vagrant.suspend() + + + def resume(self): + """Resumes the VM specified in the vagrantfile.""" + self.vagrant.resume() + + + def get_vagrant_version(self): + """Gets the vagrant version of the host. + Returns: + String: Vagrant version + """ + return self.vagrant.version() + + + def status(self): + """Gets the status of the VM specified in the vagrantfile. + Returns: + Dictionary: Status of the VM. + """ + return self.vagrant.status() + + + def get_ssh_config(self): + """Gets the config of the VM specified in the vagrantfile. + Returns: + Dictionary: Dictionary with the configuration of the VM. + """ + return self.vagrant.conf() + + + def get_instance_info(self): + """Gets the instance info. + Returns: + Dictionary: Dictionary with the parameters of the VM. + """ + return self.vagrantfile.parameters + + + def get_name(self): + """Gets the name of the VM. + Returns: + String: Name of the VM. + """ + return self.vagrantfile.vm_name diff --git a/deps/deployment/Vagrantfile.py b/deps/deployment/Vagrantfile.py new file mode 100644 index 0000000000..54f0f50d5b --- /dev/null +++ b/deps/deployment/Vagrantfile.py @@ -0,0 +1,73 @@ +# Copyright (C) 2015-2021, Wazuh Inc. +# Created by Wazuh, Inc. . +# This program is free software; you can redistribute it and/or modify it under the terms of GPLv2 +from pathlib import Path +import os +import json + +class Vagrantfile(): + """Class to handle Vagrantfile creation in runtime. This class will use a template (specified in TEMPLATE_FILE + constant) to fill the `json_box` variable. This variable will have all the needed vagrant parameters in a JSON + format. + Attributes: + TEMPLATE_FILE (str): Path where the vagrantfile_template is stored. + REPLACE_PATTERN(str): Pattern to replace inside the vagrantfile_template. + file_path (str): Path where the vagrantfile will be stored. + vm_name (str): Name of the VM. + parameters(dict): Dictionary with the parameters of the Vagrantfile. + Args: + file_path (str): Path where the vagrantfile will be stored. + box_image (str): URL for the box image or Vagrant Box. + vm_label (str): Label for the VM + vm_name (str): Name of the VM. + cpus (int): Number of CPU cores for the VM. + memory (int): Memory assigned to the VM (in MB). + system (str): Type of system (/Linux, /Windows, /Solaris....). + It MUST start with '/' to assign properly the group in VirtualBox. + private_ip (str): IP of the VM. + """ + TEMPLATE_FILE = os.path.join(Path(__file__).parent, 'vagrantfile_template.txt') + REPLACE_PATTERN = 'json_box = {}\n' + + def __init__(self, file_path, box_image, vm_label, vm_name, cpus, memory, system, private_ip): + """Constructor for the class. The method will fill a dictionary with all the information in the parameters. + + """ + self.file_path = os.path.join(file_path, 'vagrantfile') + self.vm_name = vm_name + self.parameters = { + 'box_image': box_image, + 'vm_label': vm_label, + 'cpus': cpus, + 'memory': memory, + 'system': f'/{system}', + 'private_ip': private_ip + } + + + def __str__(self): + """To str method. It will print the dictionary in JSON format.""" + return json.dumps({self.vm_name: self.parameters}) + + def read_vagrantfile_template(self): + """Function that will read the vagrantfile template located in self.TEMPLATEFILE constant + + Returns: + List: List with the content of the template vagrant template.""" + with open(self.TEMPLATE_FILE, 'r') as template_fd: + return template_fd.readlines() + + def write_vagrantfile(self): + """Replace the self.REPLACE_PATTERN line with a string with the parameters in JSON format and write the new + contents in self.file_path file.""" + read_lines = self.read_vagrantfile_template() + replace_line = read_lines.index(self.REPLACE_PATTERN) + read_lines[replace_line] = self.REPLACE_PATTERN.format(f"'{self.__str__()}'") + + with open(self.file_path, 'w') as vagrantfile_fd: + vagrantfile_fd.writelines(read_lines) + + def remove_vagrantfile(self): + """Removes the file self.file_path if it exists.""" + if os.path.exists(self.file_path): + os.remove(self.file_path) diff --git a/deps/deployment/vagrantfile_template.txt b/deps/deployment/vagrantfile_template.txt new file mode 100644 index 0000000000..da32146e5f --- /dev/null +++ b/deps/deployment/vagrantfile_template.txt @@ -0,0 +1,24 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +require 'json' + +json_box = {} +box = JSON.load(json_box) + +Vagrant.configure("2") do |config| + box.each do |vm_name, vm_parameters| + config.vm.define "#{vm_name}" do |node| + node.vm.box = "#{vm_parameters['box_image']}" + node.vm.network :private_network, ip: "#{vm_parameters['private_ip']}" + node.vm.hostname = "#{vm_name}" + node.vm.provider "virtualbox" do |vb| + vb.memory = "#{vm_parameters['memory']}" + vb.cpus = "#{vm_parameters['cpus']}" + vb.name = "#{vm_name}" + vb.customize ["setextradata", :id, "VBoxInternal/Devices/VMMDev/0/Config/GetHostTimeDisabled", 1] + vb.customize ["modifyvm", "#{vm_name}", "--groups", "#{vm_parameters['system']}"] + end + end + end +end From 13527cc47d8c7e3b0017c117a4f8af8ec0812480 Mon Sep 17 00:00:00 2001 From: Antonio Fresneda Date: Tue, 27 Jul 2021 11:28:08 +0200 Subject: [PATCH 02/10] Several enhancements in the vagrant and vagrantfile classes. --- deps/deployment/VagrantWrapper.py | 19 +++++++---------- deps/deployment/Vagrantfile.py | 35 ++++++++++++++++++------------- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/deps/deployment/VagrantWrapper.py b/deps/deployment/VagrantWrapper.py index 08cf025143..9a92dcbbab 100644 --- a/deps/deployment/VagrantWrapper.py +++ b/deps/deployment/VagrantWrapper.py @@ -6,6 +6,7 @@ import os from Instance import Instance + class VagrantWrapper(Instance): """Class to handle Vagrant operations. The class will use the Vagrantfile class to create a vagrantfile in runtime. The vagrantfile will be dumped to disk only if the up method is executed. @@ -24,6 +25,7 @@ class VagrantWrapper(Instance): vm_ip (str): IP assigned to the VM. quiet_out (Boolean): Flag to ignore the vagrant output. Defaults to True. """ + def __init__(self, vagrant_root_folder, vm_box, vm_label, vm_name, vm_cpus, vm_memory, vm_system, vm_ip, quiet_out=True): @@ -36,22 +38,18 @@ def __init__(self, vagrant_root_folder, vm_box, vm_label, vm_name, vm_cpus, vm_m self.vagrant = vagrant.Vagrant(root=box_folder, quiet_stdout=quiet_out, quiet_stderr=False) self.vagrantfile.write_vagrantfile() - def run(self): """Writes the vagrantfile and starts the VM specified in the vagrantfile.""" self.vagrant.up() - def halt(self): """Stops the VM specified in the vagrantfile.""" self.vagrant.halt() - def restart(self): """Restarts the VM specified in the vagrantfile.""" self.vagrant.restart() - def destroy(self): """Destroys the VM specified in the vagrantfile and remove the vagrantfile.""" self.vagrant.destroy() @@ -61,12 +59,10 @@ def suspend(self): """Suspends the VM specified in the vagrantfile.""" self.vagrant.suspend() - def resume(self): """Resumes the VM specified in the vagrantfile.""" self.vagrant.resume() - def get_vagrant_version(self): """Gets the vagrant version of the host. Returns: @@ -74,14 +70,15 @@ def get_vagrant_version(self): """ return self.vagrant.version() - def status(self): """Gets the status of the VM specified in the vagrantfile. + The vagrant module returns a list of namedtuples like the following + `[Status(name='ubuntu', state='not_created', provider='virtualbox')]` + but we are only interested in the `state` field. Returns: Dictionary: Status of the VM. """ - return self.vagrant.status() - + return self.vagrant.status()[0].state def get_ssh_config(self): """Gets the config of the VM specified in the vagrantfile. @@ -90,14 +87,12 @@ def get_ssh_config(self): """ return self.vagrant.conf() - def get_instance_info(self): """Gets the instance info. Returns: Dictionary: Dictionary with the parameters of the VM. """ - return self.vagrantfile.parameters - + return str(self.vagrantfile) def get_name(self): """Gets the name of the VM. diff --git a/deps/deployment/Vagrantfile.py b/deps/deployment/Vagrantfile.py index 54f0f50d5b..203632cb33 100644 --- a/deps/deployment/Vagrantfile.py +++ b/deps/deployment/Vagrantfile.py @@ -5,6 +5,7 @@ import os import json + class Vagrantfile(): """Class to handle Vagrantfile creation in runtime. This class will use a template (specified in TEMPLATE_FILE constant) to fill the `json_box` variable. This variable will have all the needed vagrant parameters in a JSON @@ -26,28 +27,31 @@ class Vagrantfile(): It MUST start with '/' to assign properly the group in VirtualBox. private_ip (str): IP of the VM. """ - TEMPLATE_FILE = os.path.join(Path(__file__).parent, 'vagrantfile_template.txt') + TEMPLATE_FILE = os.path.join( + Path(__file__).parent, 'vagrantfile_template.txt') REPLACE_PATTERN = 'json_box = {}\n' def __init__(self, file_path, box_image, vm_label, vm_name, cpus, memory, system, private_ip): - """Constructor for the class. The method will fill a dictionary with all the information in the parameters. - - """ self.file_path = os.path.join(file_path, 'vagrantfile') self.vm_name = vm_name - self.parameters = { - 'box_image': box_image, - 'vm_label': vm_label, - 'cpus': cpus, - 'memory': memory, - 'system': f'/{system}', - 'private_ip': private_ip - } - + self.box_image = box_image + self.vm_label = vm_label + self.cpus = cpus + self.memory = memory + self.system = f'/{system}' + self.private_ip = private_ip def __str__(self): """To str method. It will print the dictionary in JSON format.""" - return json.dumps({self.vm_name: self.parameters}) + parameters = { + 'box_image': self.box_image, + 'vm_label': self.vm_label, + 'cpus': self.cpus, + 'memory': self.memory, + 'system': self.system, + 'private_ip': self.private_ip + } + return json.dumps({self.vm_name: parameters}) def read_vagrantfile_template(self): """Function that will read the vagrantfile template located in self.TEMPLATEFILE constant @@ -62,7 +66,8 @@ def write_vagrantfile(self): contents in self.file_path file.""" read_lines = self.read_vagrantfile_template() replace_line = read_lines.index(self.REPLACE_PATTERN) - read_lines[replace_line] = self.REPLACE_PATTERN.format(f"'{self.__str__()}'") + read_lines[replace_line] = self.REPLACE_PATTERN.format( + f"'{self.__str__()}'") with open(self.file_path, 'w') as vagrantfile_fd: vagrantfile_fd.writelines(read_lines) From 6d71be40fdf3a802aaaa190fd06dae146b5232a2 Mon Sep 17 00:00:00 2001 From: Antonio Fresneda Date: Fri, 30 Jul 2021 12:51:00 +0200 Subject: [PATCH 03/10] Fix documentation issue. --- deps/deployment/Vagrantfile.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/deps/deployment/Vagrantfile.py b/deps/deployment/Vagrantfile.py index 203632cb33..ed240476ae 100644 --- a/deps/deployment/Vagrantfile.py +++ b/deps/deployment/Vagrantfile.py @@ -15,7 +15,14 @@ class Vagrantfile(): REPLACE_PATTERN(str): Pattern to replace inside the vagrantfile_template. file_path (str): Path where the vagrantfile will be stored. vm_name (str): Name of the VM. - parameters(dict): Dictionary with the parameters of the Vagrantfile. + file_path (str): Path where the vagrantfile will be stored. + box_image (str): URL for the box image or Vagrant Box. + vm_label (str): Label for the VM + vm_name (str): Name of the VM. + cpus (int): Number of CPU cores for the VM. + memory (int): Memory assigned to the VM (in MB). + system (str): Type of system (/Linux, /Windows, /Solaris....). + It MUST start with '/' to assign properly the group in VirtualBox. Args: file_path (str): Path where the vagrantfile will be stored. box_image (str): URL for the box image or Vagrant Box. From d2226c64a23908145dbb1d4561a5b366044e42a7 Mon Sep 17 00:00:00 2001 From: Antonio Fresneda Date: Fri, 23 Jul 2021 12:57:15 +0200 Subject: [PATCH 04/10] Add Instance, InstanceHandler and deployment script. --- deps/deployment/Instance.py | 41 ++++++++++++++++ deps/deployment/InstanceHandler.py | 77 ++++++++++++++++++++++++++++++ deps/deployment/conf.yaml | 42 ++++++++++++++++ deps/deployment/qa_deployment.py | 41 ++++++++++++++++ 4 files changed, 201 insertions(+) create mode 100644 deps/deployment/Instance.py create mode 100644 deps/deployment/InstanceHandler.py create mode 100644 deps/deployment/conf.yaml create mode 100644 deps/deployment/qa_deployment.py diff --git a/deps/deployment/Instance.py b/deps/deployment/Instance.py new file mode 100644 index 0000000000..b2ab1e0154 --- /dev/null +++ b/deps/deployment/Instance.py @@ -0,0 +1,41 @@ +# Copyright (C) 2015-2021, Wazuh Inc. +# Created by Wazuh, Inc. . +# This program is free software; you can redistribute it and/or modify it under the terms of GPLv2 +from abc import ABC, abstractmethod + + +class Instance(ABC): + """Abstract class to hold common methods for instance handling""" + @abstractmethod + def run(self): + """Method to start the instance.""" + pass + + @abstractmethod + def restart(self): + """Method to restart the instance.""" + pass + + @abstractmethod + def halt(self): + """Method to stop the instance.""" + pass + + @abstractmethod + def destroy(self): + """Method to destroy the instance.""" + pass + + @abstractmethod + def get_instance_info(self): + """Method to get the instance information.""" + pass + + @abstractmethod + def get_name(self): + """Method to get the instance name.""" + pass + + @abstractmethod + def status(self): + """Method to get the instance status.""" diff --git a/deps/deployment/InstanceHandler.py b/deps/deployment/InstanceHandler.py new file mode 100644 index 0000000000..eb92136c28 --- /dev/null +++ b/deps/deployment/InstanceHandler.py @@ -0,0 +1,77 @@ +# Copyright (C) 2015-2021, Wazuh Inc. +# Created by Wazuh, Inc. . +# This program is free software; you can redistribute it and/or modify it under the terms of GPLv2 +from DockerWrapper import DockerWrapper +from VagrantWrapper import VagrantWrapper + + +class InstanceHandler: + """Class to handle multiples instances objects. + Attributes: + instances (list): List with the instances to handle. + Args: + vm_list (dict): Dictionary with the information of the instances. Must follow the format of the yaml template. + """ + instances = [] + + def __init__(self, vm_list): + for host in vm_list: + for provider in vm_list[host]['provider']: + data = vm_list[host]['provider'][provider] + if provider == 'vagrant': + quiet_out = True if 'quiet_out' not in data.keys() else data['quiet_out'] + if not data['enabled']: + continue + vagrant_instance = VagrantWrapper(data['vagrantfile_path'], data['vagrant_box'], data['label'], + data['vm_name'], data['vm_cpu'], data['vm_memory'], + data['vm_system'], data['vm_ip'], quiet_out) + self.instances.append(vagrant_instance) + + elif provider == 'docker': + + docker_instance = DockerWrapper(data['dockerfile_path'], data['name'], data['remove'], + data['ports'], data['detach'], data['stdout'], data['stderr']) + self.instances.append(docker_instance) + + def run(self): + """Executes the run method on every configured instance.""" + for instance in self.instances: + instance.run() + + def halt(self): + """Executes the 'halt' method on every configured instance.""" + for instance in self.instances: + instance.halt() + + def restart(self): + """Executes the 'restart' method on every configured instance.""" + for instance in self.instances: + instance.restart() + + def destroy(self): + """Executes the 'destroy' method on every configured instance.""" + for instance in self.instances: + instance.destroy() + + def status(self): + """Executes the 'status' method on every configured instance. + + Returns: + Dictionary: Contains the status for each configured instance. + """ + status = {} + for instance in self.instances: + status[instance.get_name()] = instance.status() + + return status + + def get_instances_info(self): + """Get information about the information for all the configured instances. + Returns: + Dictionary: Dictionary with the information for each configured instance. + """ + info = {} + for instance in self.instances: + info[instance.get_name()] = instance.get_instance_info() + + return info diff --git a/deps/deployment/conf.yaml b/deps/deployment/conf.yaml new file mode 100644 index 0000000000..7acab86e08 --- /dev/null +++ b/deps/deployment/conf.yaml @@ -0,0 +1,42 @@ +host1: + provider: + vagrant: + enabled: True + vagrantfile_path: '/tmp' + vagrant_box: 'bento/ubuntu-20.04' + vm_memory: 1024 + vm_cpu: 1 + vm_name: 'ubuntu' + vm_system: 'Linux' + os: Ubuntu + label: 'Label' + vm_ip: '10.2.0.3' + quiet_out: False +host2: + provider: + vagrant: + enabled: True + vagrantfile_path: '/tmp' + vagrant_box: 'jacqinthebox/windowsserver2016' + vm_memory: 1024 + vm_cpu: 1 + vm_name: 'Windows2016' + vm_system: 'Windows' + os: Windows + label: 'Label' + vm_ip: '10.2.0.4' + quiet_out: False +host3: + provider: + vagrant: + enabled: False + vagrantfile_path: '/tmp' + vagrant_box: 'https://s3.amazonaws.com/ci.wazuh.com/vms/vagrant/solaris/11/i386/solaris11.3.box' + vm_memory: 1024 + vm_cpu: 1 + vm_name: 'solaris11' + vm_system: 'Solaris' + os: solaris11 + label: 'Label' + vm_ip: '10.2.0.5' + quiet_out: False diff --git a/deps/deployment/qa_deployment.py b/deps/deployment/qa_deployment.py new file mode 100644 index 0000000000..bf040c86cb --- /dev/null +++ b/deps/deployment/qa_deployment.py @@ -0,0 +1,41 @@ +# Copyright (C) 2015-2021, Wazuh Inc. +# Created by Wazuh, Inc. . +# This program is free software; you can redistribute it and/or modify it under the terms of GPLv2 +import argparse +import os +from InstanceHandler import InstanceHandler +import yaml + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + yaml_config = {} + parser.add_argument('--config', '-c', type=str, action='store', required=True, + help='Path to the configuration file.') + parser.add_argument('--action', '-a', type=str, action='store', required=True, + choices=['run', 'halt', 'status', 'info', 'destroy'], + help='Action to perform') + + arguments = parser.parse_args() + + assert os.path.exists( + arguments.config), f"{arguments.config} file doesn't exists" + + with open(arguments.config) as config_file_fd: + yaml_config = yaml.safe_load(config_file_fd) + + instance_handler = InstanceHandler(yaml_config) + + if arguments.action == 'run': + instance_handler.run() + + elif arguments.action == 'halt': + instance_handler.halt() + + elif arguments.action == 'status': + print(instance_handler.status()) + + elif arguments.action == 'info': + print(instance_handler.get_instances_info()) + + elif arguments.action == 'destroy': + instance_handler.destroy() From e4002951cc381c2921502129ba79c24d1e30ff1c Mon Sep 17 00:00:00 2001 From: Antonio Fresneda Date: Fri, 30 Jul 2021 11:11:52 +0200 Subject: [PATCH 05/10] Minor change in InstanceHandler constructor --- deps/deployment/InstanceHandler.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/deps/deployment/InstanceHandler.py b/deps/deployment/InstanceHandler.py index eb92136c28..07afb94f3f 100644 --- a/deps/deployment/InstanceHandler.py +++ b/deps/deployment/InstanceHandler.py @@ -18,17 +18,17 @@ def __init__(self, vm_list): for host in vm_list: for provider in vm_list[host]['provider']: data = vm_list[host]['provider'][provider] + if not data['enabled']: + continue + if provider == 'vagrant': quiet_out = True if 'quiet_out' not in data.keys() else data['quiet_out'] - if not data['enabled']: - continue vagrant_instance = VagrantWrapper(data['vagrantfile_path'], data['vagrant_box'], data['label'], data['vm_name'], data['vm_cpu'], data['vm_memory'], data['vm_system'], data['vm_ip'], quiet_out) self.instances.append(vagrant_instance) elif provider == 'docker': - docker_instance = DockerWrapper(data['dockerfile_path'], data['name'], data['remove'], data['ports'], data['detach'], data['stdout'], data['stderr']) self.instances.append(docker_instance) From cb68b5eed8a7faf19292893a201fd261d26dd6b1 Mon Sep 17 00:00:00 2001 From: Antonio Fresneda Date: Fri, 30 Jul 2021 14:59:06 +0200 Subject: [PATCH 06/10] Add docker wrapper class and conf example. Also includes the necessary changes in the qa_deployment script --- deps/deployment/DockerWrapper.py | 65 ++++++++++++++++++++++++++++++++ deps/deployment/conf.yaml | 15 +++++++- 2 files changed, 79 insertions(+), 1 deletion(-) create mode 100644 deps/deployment/DockerWrapper.py diff --git a/deps/deployment/DockerWrapper.py b/deps/deployment/DockerWrapper.py new file mode 100644 index 0000000000..261828b50d --- /dev/null +++ b/deps/deployment/DockerWrapper.py @@ -0,0 +1,65 @@ +from os import remove +import docker +from Instance import Instance + + +class DockerWrapper(Instance): + def __init__(self, dockerfile_path, name, remove, ports=None, detach=True, stdout=False, stderr=False): + self.docker_client = docker.from_env() + self.dockerfile_path = dockerfile_path + self.name = name + self.remove = remove + self.detach = detach + self.ports = ports + + if self.detach: + self.stdout = stdout + self.stderr = stderr + else: + self.stdout = True + self.stderr = True + + self.image = self.docker_client.images.build(path=self.dockerfile_path)[0] + + + def get_container(self): + return self.docker_client.containers.get(self.name) + + + def run(self): + try: + existing_container = self.docker_client.containers.get(self.name) + existing_container.start() + + except docker.errors.NotFound: + self.docker_client.containers.run(image=self.image, name=self.name, ports=self.ports, + remove=self.remove, detach=self.detach, stdout=self.stdout, + stderr=self.stderr) + + def restart(self): + self.get_container().restart() + + + def halt(self): + self.get_container().stop() + + + def destroy(self, remove_image=False): + self.get_container().remove() + if remove_image: + self.docker_client.images.remove(image=self.image.id, force=True) + + + def get_instance_info(self): + return self.parameters + + + def get_name(self): + return self.name + + def status(self): + try: + status = self.get_container().status + except docker.errors.NotFound: + status = 'Not created' + return status diff --git a/deps/deployment/conf.yaml b/deps/deployment/conf.yaml index 7acab86e08..1aaf29b403 100644 --- a/deps/deployment/conf.yaml +++ b/deps/deployment/conf.yaml @@ -15,7 +15,7 @@ host1: host2: provider: vagrant: - enabled: True + enabled: False vagrantfile_path: '/tmp' vagrant_box: 'jacqinthebox/windowsserver2016' vm_memory: 1024 @@ -40,3 +40,16 @@ host3: label: 'Label' vm_ip: '10.2.0.5' quiet_out: False + + +host4: + provider: + docker: + enabled: True + dockerfile_path: '/tmp/prueba' + name: 'test' + remove: False + ports: + detach: False + stdout: True + stderr: True From 6707c1ecef157ec0bd86b6302c17fe34227c4904 Mon Sep 17 00:00:00 2001 From: Antonio Fresneda Date: Tue, 27 Jul 2021 11:26:14 +0200 Subject: [PATCH 07/10] Add documentation. --- deps/deployment/DockerWrapper.py | 85 +++++++++++++++++++++++++------- 1 file changed, 68 insertions(+), 17 deletions(-) diff --git a/deps/deployment/DockerWrapper.py b/deps/deployment/DockerWrapper.py index 261828b50d..534ffbf8c5 100644 --- a/deps/deployment/DockerWrapper.py +++ b/deps/deployment/DockerWrapper.py @@ -1,9 +1,32 @@ -from os import remove import docker from Instance import Instance +from json import dumps class DockerWrapper(Instance): + """Class to handle docker operations. This class uses the docker python SDK to read a dockerfile and create + the image and container. + + Attributes: + dockerfile_path (str): Path where the Dockerfile is stored. + name (str): Container's name. + remove (bool): Remove the container after it has finished. + detach (bool): Run container in background. + ports (dict): Ports to bind inside the container. + The keys of the dictionary are the ports to bind inside the container and the values of the + dictionary are the corresponding ports to open on the host. + stdout (bool): Return stdout logs when detach is False. + stderr (bool): Return stderr logs when detach is False. + + Args: + dockerfile_path (str): Value to set dockerfile_path attribute. + name (str): Value to set name attribute. + remove (bool): Value to set remove attribute. + detach (bool): Value to set detach attribute. + ports (dict): Value to set ports attribute. + stdout (bool): Value to set stdout attribute. + stderr (bool): Value to set stderr attribute. + """ def __init__(self, dockerfile_path, name, remove, ports=None, detach=True, stdout=False, stderr=False): self.docker_client = docker.from_env() self.dockerfile_path = dockerfile_path @@ -21,45 +44,73 @@ def __init__(self, dockerfile_path, name, remove, ports=None, detach=True, stdou self.image = self.docker_client.images.build(path=self.dockerfile_path)[0] - def get_container(self): - return self.docker_client.containers.get(self.name) + """Function to get the container using the name attribute: + Returns: + Container: Container object with the container info. - def run(self): - try: - existing_container = self.docker_client.containers.get(self.name) - existing_container.start() + Raises: + docker.errors.NotFound: If the container does not exist. + docker.errors.APIError: If the server returns an error. + """ + return self.docker_client.containers.get(self.name) - except docker.errors.NotFound: - self.docker_client.containers.run(image=self.image, name=self.name, ports=self.ports, - remove=self.remove, detach=self.detach, stdout=self.stdout, - stderr=self.stderr) + def run(self): + self.docker_client.containers.run(image=self.image, name=self.name, ports=self.ports, + remove=self.remove, detach=self.detach, stdout=self.stdout, + stderr=self.stderr) def restart(self): + """Restart the container. + Raises: + docker.errors.APIError: If the server returns an error. + """ self.get_container().restart() - def halt(self): + """Stops the container. + Raises: + docker.errors.APIError: If the server returns an error. + """ self.get_container().stop() - def destroy(self, remove_image=False): + """Removes the container + Args: + remove_image(bool): Remove the docker image too. Defaults to False. + Raises: + docker.errors.APIError: If the server returns an error. + """ self.get_container().remove() if remove_image: self.docker_client.images.remove(image=self.image.id, force=True) - def get_instance_info(self): - return self.parameters - + """Get the parameters information. + Returns + str: String in JSON format with the parameters of the class. + """ + return dumps({'name': self.name, 'parameters': { + 'dockerfile_path': self.dockerfile_path, 'remove': self.remove, + 'detach': self.detach, 'ports': self.ports, 'stderr': self.stderr, + 'stdout': self.stdout} + }) def get_name(self): + """Get the name of the container. + Returns + str: String with the name of the container. + """ return self.name def status(self): + """Get the status of the container. + Returns: + str: String with the status of the container (running, exited, not created, etc). + """ try: status = self.get_container().status except docker.errors.NotFound: - status = 'Not created' + status = 'not_created' return status From fb6d5b3d8410574c07048816ba1b139f19336c18 Mon Sep 17 00:00:00 2001 From: Antonio Fresneda Date: Thu, 29 Jul 2021 08:55:49 +0200 Subject: [PATCH 08/10] Add dockerfiles and entrypoint script. --- deps/deployment/dockerfiles/ALDockerfile | 17 ++++++++++++++ deps/deployment/dockerfiles/UBDockerfile | 27 +++++++++++++++++++++++ deps/deployment/dockerfiles/entrypoint.sh | 3 +++ 3 files changed, 47 insertions(+) create mode 100644 deps/deployment/dockerfiles/ALDockerfile create mode 100644 deps/deployment/dockerfiles/UBDockerfile create mode 100755 deps/deployment/dockerfiles/entrypoint.sh diff --git a/deps/deployment/dockerfiles/ALDockerfile b/deps/deployment/dockerfiles/ALDockerfile new file mode 100644 index 0000000000..9caf145359 --- /dev/null +++ b/deps/deployment/dockerfiles/ALDockerfile @@ -0,0 +1,17 @@ +FROM amazonlinux:2.0.20200602.0 + +RUN yum update -y && \ + yum install -y \ + openssh-server \ + nano \ + openssl \ + sudo \ + gcc \ + git \ + python3 \ + python3-devel + +ADD entrypoint.sh /usr/bin/entrypoint.sh +RUN chmod +x /usr/bin/entrypoint.sh + +ENTRYPOINT ["/usr/bin/entrypoint.sh"] diff --git a/deps/deployment/dockerfiles/UBDockerfile b/deps/deployment/dockerfiles/UBDockerfile new file mode 100644 index 0000000000..e1a6a2b340 --- /dev/null +++ b/deps/deployment/dockerfiles/UBDockerfile @@ -0,0 +1,27 @@ +FROM ubuntu:18.04 + +RUN apt-get update && \ + apt-get install -y \ + software-properties-common \ + openssl \ + curl \ + apt-transport-https \ + lsb-release \ + gnupg2 \ + make \ + libc6-dev \ + gcc \ + g++ \ + python \ + policycoreutils \ + automake \ + autoconf \ + libtool \ + libssl-dev \ + git \ + wget + +ADD entrypoint.sh /usr/bin/entrypoint.sh +RUN chmod +x /usr/bin/entrypoint.sh + +ENTRYPOINT ["/usr/bin/entrypoint.sh"] diff --git a/deps/deployment/dockerfiles/entrypoint.sh b/deps/deployment/dockerfiles/entrypoint.sh new file mode 100755 index 0000000000..1fb176615f --- /dev/null +++ b/deps/deployment/dockerfiles/entrypoint.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +tail -f /dev/null From 85ab6f6d955c552cf3fe3cf9d9113b29b7c9187a Mon Sep 17 00:00:00 2001 From: Antonio Fresneda Date: Tue, 3 Aug 2021 12:35:51 +0200 Subject: [PATCH 09/10] Several enhancements into the deployment phase: - Allow optional arguments in docker configuration. - Rename InstanceHandler class by QAInfraestructure. --- .../{InstanceHandler.py => QAInfraestructure.py} | 11 ++++++++--- deps/deployment/conf.yaml | 4 +--- deps/deployment/qa_deployment.py | 4 ++-- 3 files changed, 11 insertions(+), 8 deletions(-) rename deps/deployment/{InstanceHandler.py => QAInfraestructure.py} (84%) diff --git a/deps/deployment/InstanceHandler.py b/deps/deployment/QAInfraestructure.py similarity index 84% rename from deps/deployment/InstanceHandler.py rename to deps/deployment/QAInfraestructure.py index 07afb94f3f..39fbd0fdb8 100644 --- a/deps/deployment/InstanceHandler.py +++ b/deps/deployment/QAInfraestructure.py @@ -5,7 +5,7 @@ from VagrantWrapper import VagrantWrapper -class InstanceHandler: +class QAInfraestructure: """Class to handle multiples instances objects. Attributes: instances (list): List with the instances to handle. @@ -22,15 +22,20 @@ def __init__(self, vm_list): continue if provider == 'vagrant': - quiet_out = True if 'quiet_out' not in data.keys() else data['quiet_out'] + quiet_out = True if 'quiet_out' not in data else data['quiet_out'] vagrant_instance = VagrantWrapper(data['vagrantfile_path'], data['vagrant_box'], data['label'], data['vm_name'], data['vm_cpu'], data['vm_memory'], data['vm_system'], data['vm_ip'], quiet_out) self.instances.append(vagrant_instance) elif provider == 'docker': + _ports = None if 'ports' not in data else data['ports'] + _detach = True if 'detach' not in data else data['detach'] + _stdout = False if 'stdout' not in data else data['stdout'] + _stderr = False if 'stderr' not in data else data['stderr'] + docker_instance = DockerWrapper(data['dockerfile_path'], data['name'], data['remove'], - data['ports'], data['detach'], data['stdout'], data['stderr']) + _ports, _detach, _stdout, _stderr) self.instances.append(docker_instance) def run(self): diff --git a/deps/deployment/conf.yaml b/deps/deployment/conf.yaml index 1aaf29b403..a016cda690 100644 --- a/deps/deployment/conf.yaml +++ b/deps/deployment/conf.yaml @@ -50,6 +50,4 @@ host4: name: 'test' remove: False ports: - detach: False - stdout: True - stderr: True + detach: True diff --git a/deps/deployment/qa_deployment.py b/deps/deployment/qa_deployment.py index bf040c86cb..cd8c8410fe 100644 --- a/deps/deployment/qa_deployment.py +++ b/deps/deployment/qa_deployment.py @@ -3,7 +3,7 @@ # This program is free software; you can redistribute it and/or modify it under the terms of GPLv2 import argparse import os -from InstanceHandler import InstanceHandler +from QAInfraestructure import QAInfraestructure import yaml if __name__ == '__main__': @@ -23,7 +23,7 @@ with open(arguments.config) as config_file_fd: yaml_config = yaml.safe_load(config_file_fd) - instance_handler = InstanceHandler(yaml_config) + instance_handler = QAInfraestructure(yaml_config) if arguments.action == 'run': instance_handler.run() From 45fe375ea82b95359cb19062f6d0fd21477aa8d7 Mon Sep 17 00:00:00 2001 From: Antonio Fresneda Date: Tue, 3 Aug 2021 14:40:56 +0200 Subject: [PATCH 10/10] Move deployment module to qa_ctl folder. --- deps/deployment/conf.yaml | 53 ------------------- deps/deployment/qa_deployment.py | 41 -------------- .../qa_ctl}/deployment/DockerWrapper.py | 0 .../qa_ctl}/deployment/Instance.py | 0 .../qa_ctl}/deployment/QAInfraestructure.py | 0 .../qa_ctl}/deployment/VagrantWrapper.py | 0 .../qa_ctl}/deployment/Vagrantfile.py | 0 .../qa_ctl/deployment/__init__.py | 0 .../deployment/dockerfiles/ALDockerfile | 0 .../deployment/dockerfiles/UBDockerfile | 0 .../deployment/dockerfiles/entrypoint.sh | 0 .../deployment/vagrantfile_template.txt | 0 12 files changed, 94 deletions(-) delete mode 100644 deps/deployment/conf.yaml delete mode 100644 deps/deployment/qa_deployment.py rename deps/{ => wazuh_testing/wazuh_testing/qa_ctl}/deployment/DockerWrapper.py (100%) rename deps/{ => wazuh_testing/wazuh_testing/qa_ctl}/deployment/Instance.py (100%) rename deps/{ => wazuh_testing/wazuh_testing/qa_ctl}/deployment/QAInfraestructure.py (100%) rename deps/{ => wazuh_testing/wazuh_testing/qa_ctl}/deployment/VagrantWrapper.py (100%) rename deps/{ => wazuh_testing/wazuh_testing/qa_ctl}/deployment/Vagrantfile.py (100%) create mode 100644 deps/wazuh_testing/wazuh_testing/qa_ctl/deployment/__init__.py rename deps/{ => wazuh_testing/wazuh_testing/qa_ctl}/deployment/dockerfiles/ALDockerfile (100%) rename deps/{ => wazuh_testing/wazuh_testing/qa_ctl}/deployment/dockerfiles/UBDockerfile (100%) rename deps/{ => wazuh_testing/wazuh_testing/qa_ctl}/deployment/dockerfiles/entrypoint.sh (100%) rename deps/{ => wazuh_testing/wazuh_testing/qa_ctl}/deployment/vagrantfile_template.txt (100%) diff --git a/deps/deployment/conf.yaml b/deps/deployment/conf.yaml deleted file mode 100644 index a016cda690..0000000000 --- a/deps/deployment/conf.yaml +++ /dev/null @@ -1,53 +0,0 @@ -host1: - provider: - vagrant: - enabled: True - vagrantfile_path: '/tmp' - vagrant_box: 'bento/ubuntu-20.04' - vm_memory: 1024 - vm_cpu: 1 - vm_name: 'ubuntu' - vm_system: 'Linux' - os: Ubuntu - label: 'Label' - vm_ip: '10.2.0.3' - quiet_out: False -host2: - provider: - vagrant: - enabled: False - vagrantfile_path: '/tmp' - vagrant_box: 'jacqinthebox/windowsserver2016' - vm_memory: 1024 - vm_cpu: 1 - vm_name: 'Windows2016' - vm_system: 'Windows' - os: Windows - label: 'Label' - vm_ip: '10.2.0.4' - quiet_out: False -host3: - provider: - vagrant: - enabled: False - vagrantfile_path: '/tmp' - vagrant_box: 'https://s3.amazonaws.com/ci.wazuh.com/vms/vagrant/solaris/11/i386/solaris11.3.box' - vm_memory: 1024 - vm_cpu: 1 - vm_name: 'solaris11' - vm_system: 'Solaris' - os: solaris11 - label: 'Label' - vm_ip: '10.2.0.5' - quiet_out: False - - -host4: - provider: - docker: - enabled: True - dockerfile_path: '/tmp/prueba' - name: 'test' - remove: False - ports: - detach: True diff --git a/deps/deployment/qa_deployment.py b/deps/deployment/qa_deployment.py deleted file mode 100644 index cd8c8410fe..0000000000 --- a/deps/deployment/qa_deployment.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright (C) 2015-2021, Wazuh Inc. -# Created by Wazuh, Inc. . -# This program is free software; you can redistribute it and/or modify it under the terms of GPLv2 -import argparse -import os -from QAInfraestructure import QAInfraestructure -import yaml - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - yaml_config = {} - parser.add_argument('--config', '-c', type=str, action='store', required=True, - help='Path to the configuration file.') - parser.add_argument('--action', '-a', type=str, action='store', required=True, - choices=['run', 'halt', 'status', 'info', 'destroy'], - help='Action to perform') - - arguments = parser.parse_args() - - assert os.path.exists( - arguments.config), f"{arguments.config} file doesn't exists" - - with open(arguments.config) as config_file_fd: - yaml_config = yaml.safe_load(config_file_fd) - - instance_handler = QAInfraestructure(yaml_config) - - if arguments.action == 'run': - instance_handler.run() - - elif arguments.action == 'halt': - instance_handler.halt() - - elif arguments.action == 'status': - print(instance_handler.status()) - - elif arguments.action == 'info': - print(instance_handler.get_instances_info()) - - elif arguments.action == 'destroy': - instance_handler.destroy() diff --git a/deps/deployment/DockerWrapper.py b/deps/wazuh_testing/wazuh_testing/qa_ctl/deployment/DockerWrapper.py similarity index 100% rename from deps/deployment/DockerWrapper.py rename to deps/wazuh_testing/wazuh_testing/qa_ctl/deployment/DockerWrapper.py diff --git a/deps/deployment/Instance.py b/deps/wazuh_testing/wazuh_testing/qa_ctl/deployment/Instance.py similarity index 100% rename from deps/deployment/Instance.py rename to deps/wazuh_testing/wazuh_testing/qa_ctl/deployment/Instance.py diff --git a/deps/deployment/QAInfraestructure.py b/deps/wazuh_testing/wazuh_testing/qa_ctl/deployment/QAInfraestructure.py similarity index 100% rename from deps/deployment/QAInfraestructure.py rename to deps/wazuh_testing/wazuh_testing/qa_ctl/deployment/QAInfraestructure.py diff --git a/deps/deployment/VagrantWrapper.py b/deps/wazuh_testing/wazuh_testing/qa_ctl/deployment/VagrantWrapper.py similarity index 100% rename from deps/deployment/VagrantWrapper.py rename to deps/wazuh_testing/wazuh_testing/qa_ctl/deployment/VagrantWrapper.py diff --git a/deps/deployment/Vagrantfile.py b/deps/wazuh_testing/wazuh_testing/qa_ctl/deployment/Vagrantfile.py similarity index 100% rename from deps/deployment/Vagrantfile.py rename to deps/wazuh_testing/wazuh_testing/qa_ctl/deployment/Vagrantfile.py diff --git a/deps/wazuh_testing/wazuh_testing/qa_ctl/deployment/__init__.py b/deps/wazuh_testing/wazuh_testing/qa_ctl/deployment/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/deps/deployment/dockerfiles/ALDockerfile b/deps/wazuh_testing/wazuh_testing/qa_ctl/deployment/dockerfiles/ALDockerfile similarity index 100% rename from deps/deployment/dockerfiles/ALDockerfile rename to deps/wazuh_testing/wazuh_testing/qa_ctl/deployment/dockerfiles/ALDockerfile diff --git a/deps/deployment/dockerfiles/UBDockerfile b/deps/wazuh_testing/wazuh_testing/qa_ctl/deployment/dockerfiles/UBDockerfile similarity index 100% rename from deps/deployment/dockerfiles/UBDockerfile rename to deps/wazuh_testing/wazuh_testing/qa_ctl/deployment/dockerfiles/UBDockerfile diff --git a/deps/deployment/dockerfiles/entrypoint.sh b/deps/wazuh_testing/wazuh_testing/qa_ctl/deployment/dockerfiles/entrypoint.sh similarity index 100% rename from deps/deployment/dockerfiles/entrypoint.sh rename to deps/wazuh_testing/wazuh_testing/qa_ctl/deployment/dockerfiles/entrypoint.sh diff --git a/deps/deployment/vagrantfile_template.txt b/deps/wazuh_testing/wazuh_testing/qa_ctl/deployment/vagrantfile_template.txt similarity index 100% rename from deps/deployment/vagrantfile_template.txt rename to deps/wazuh_testing/wazuh_testing/qa_ctl/deployment/vagrantfile_template.txt