Merge pull request #397 from dshulyak/multi_transports

rsync + raw_ssh by default

Dockerfile

@@ -7,9 +7,8 @@ ADD resources /resources
 ADD templates /templates
 ADD run.sh /run.sh
 
-RUN apt-get update
-RUN apt-get install -y python python-dev python-distribute python-pip \
-    libyaml-dev vim libffi-dev libssl-dev git
+RUN apt-get upgrade && apt-get update
+RUN apt-get install -y python python-dev python-distribute python-pip openssh-client rsync libyaml-dev vim libffi-dev libssl-dev git
 RUN pip install ansible
 
 RUN pip install git+https://github.com/Mirantis/solar.git

bootstrap/playbooks/build-main.yaml

@@ -3,10 +3,12 @@
 - name: Main build script
   hosts: all
   sudo: yes
+  vars:
+    ssh_ip_mask: "10.0.0.*"
   tasks:
     - include: tasks/base.yaml
     - include: tasks/puppet.yaml
     - include: tasks/docker.yaml
-    #- include: celery.yaml tags=['master'] celery_dir=/var/run/celery
     - include: tasks/cloud_archive.yaml
-    #- include: tasks/mos.yaml
+    - include: tasks/ssh_conf.yaml
+

bootstrap/playbooks/celery.yaml

@@ -18,6 +18,4 @@
     - shell: celery multi start 2 -A solar.orchestration.runner -P:2 prefork -c:1 1 -c:2 3 -Q:1 scheduler,system_log -Q:2 celery,{{ hostname.stdout }}
              chdir={{ celery_dir }}
       tags: [master]
-    - shell: celery multi start 1 -A solar.orchestration.runner -Q:1  {{ hostname.stdout }}
-             chdir={{ celery_dir }}
-      tags: [slave]
+

bootstrap/playbooks/files/ssh_conf

@@ -0,0 +1,2 @@
+Host {{ssh_ip_mask}}
+  StrictHostKeyChecking no

bootstrap/playbooks/solar.yaml

@@ -2,12 +2,15 @@
 
 - hosts: all
   sudo: yes
+  vars:
+    ssh_ip_mask: "10.0.0.*"
   tasks:
     # upgrade pbr first, old version throws strange errors
     - shell: pip install pbr -U
     # Setup development env for solar
     - shell: pip install -e . chdir=/vagrant
     - shell: pip install git+git://github.com/Mirantis/solar-agent.git
+    - include: tasks/ssh_conf.yaml
 
 - hosts: all
   tasks:

bootstrap/playbooks/tasks/ssh_conf.yaml

@@ -0,0 +1,3 @@
+---
+
+- template: src=files/ssh_conf dest=/root/.ssh/config

doc/source/transports.rst

@@ -31,6 +31,16 @@ Currently there are following sync transports available:
 * solar_agent
 * torrent
 
+Ssh host key checking
+---------------------
+Solar wont disable strict host key checking by default, so before working
+with solar ensure that strict host key checking is disabled, or all target hosts added to .ssh/known_hosts file.
+
+Example of .ssh/config ::
+
+  Host 10.0.0.*
+    StrictHostKeyChecking no
+
 Run transport
 -------------
 

docker-compose.yml

@@ -8,6 +8,8 @@ solar-celery:
     - /vagrant/templates:/vagrant/templates
     - /vagrant/resources:/vagrant/resources
     - /vagrant/library:/vagrant/library
+    - ~/.ssh:/root/.ssh
+    - ./bootstrap/playbooks/celery.yaml:/celery.yaml
   environment:
     - REDIS_HOST=redis
     - REDIS_PORT=6379

examples/hosts_file/hosts.py

@@ -10,12 +10,12 @@
 def run():
     ModelMeta.remove_all()
 
-    resources = vr.create('nodes', 'templates/nodes_with_transports.yaml', {'count': 2})
-    nodes = [x for x in resources if x.name.startswith('node')]
-    node1, node2 = nodes
+    resources = vr.create('nodes', 'templates/nodes.yaml', {'count': 2})
+
+    node1, node2 = [x for x in resources if x.name.startswith('node')]
+    hosts1, hosts2 = [x for x in resources
+                      if x.name.startswith('hosts_file')]
 
-    hosts1 = vr.create('hosts_file1', 'resources/hosts_file', {})[0]
-    hosts2 = vr.create('hosts_file2', 'resources/hosts_file', {})[0]
     node1.connect(hosts1, {
         'name': 'hosts:name',
         'ip': 'hosts:ip',
@@ -36,5 +36,4 @@ def run():
         'ip': 'hosts:ip',
     })
 
-
 run()

resources/transport_rsync/meta.yaml

@@ -0,0 +1,23 @@
+id: transport_rsync
+input:
+  key:
+    schema: str!
+    value:
+  user:
+    schema: str!
+    value:
+  name:
+    schema: str!
+    value: rsync
+  location_id:
+    schema: str
+    value:
+    reverse: True
+    is_own: False
+  transports_id:
+    schema: str
+    value:
+    is_emit: False
+  port:
+    schema: int
+    value: 3579

run.sh

@@ -6,6 +6,6 @@ if [ -d /solar ]; then
 fi
 
 #used only to start celery on docker
-ansible-playbook -v -i "localhost," -c local /celery.yaml --skip-tags slave,stop
+ansible-playbook -v -i "localhost," -c local /celery.yaml --skip-tags install
 
 tail -f /var/run/celery/*.log

solar/core/handlers/ansible_template.py

@@ -19,7 +19,6 @@
 from solar.core.handlers.base import SOLAR_TEMP_LOCAL_LOCATION
 from solar.core.handlers.base import TempFileHandler
 from solar.core.log import log
-from solar import errors
 
 # otherwise fabric will sys.exit(1) in case of errors
 env.warn_only = True
@@ -52,15 +51,8 @@ def action(self, resource, action_name):
                      '-i', remote_inventory_file, remote_playbook_file]
         log.debug('EXECUTING: %s', ' '.join(call_args))
 
-        out = self.transport_run.run(resource, *call_args)
-        log.debug(out)
-        if out.failed:
-            raise errors.SolarError(out)
-
-        # with fabric_api.shell_env(ANSIBLE_HOST_KEY_CHECKING='False'):
-        #     out = fabric_api.local(' '.join(call_args), capture=True)
-        # if out.failed:
-        #     raise errors.SolarError(out)
+        rst = self.transport_run.run(resource, *call_args)
+        self.verify_run_result(call_args, rst)
 
     def _create_inventory(self, r):
         directory = self.dirs[r.name]

solar/core/handlers/base.py

@@ -21,6 +21,7 @@
 from solar.core.log import log
 from solar.core.transports.ssh import SSHRunTransport
 from solar.core.transports.ssh import SSHSyncTransport
+from solar import errors
 from solar import utils
 
 
@@ -42,6 +43,13 @@ def __init__(self, resources, handlers=None):
         self.transport_sync.bind_with(self.transport_run)
         self.transport_run.bind_with(self.transport_sync)
 
+    def verify_run_result(self, cmd, result):
+        rc, out, err = result
+        log.debug('CMD %r RC %s OUT %s ERR %s', cmd, rc, out, err)
+        if rc:
+            message = 'CMD %r failed RC %s ERR %s' % (cmd, rc, err)
+            raise errors.SolarError(message)
+
     def __enter__(self):
         return self
 

solar/core/handlers/puppet.py

@@ -46,7 +46,7 @@ def action(self, resource, action_name):
             cmd_args.append('--modulepath={}'.format(
                 resource.args['puppet_modules']))
 
-        cmd = self.transport_run.run(
+        rc, out, err = self.transport_run.run(
             resource,
             *cmd_args,
             env={
@@ -55,12 +55,12 @@ def action(self, resource, action_name):
             use_sudo=True,
             warn_only=True
         )
+        log.debug('CMD %r RC %s OUT %s ERR %s', cmd_args, rc, out, err)
         # 0 - no changes, 2 - successfull changes
-        if cmd.return_code not in [0, 2]:
+        if rc not in [0, 2]:
             raise errors.SolarError(
-                'Puppet for {} failed with {}'.format(
-                    resource.name, cmd.return_code))
-        return cmd
+                'Puppet for {} failed with RC {}'.format(
+                    resource.name, rc))
 
     def _make_args(self, resource):
         return {resource.name: {'input': resource.args}}

solar/core/handlers/shell.py

@@ -18,7 +18,6 @@
 from solar.core.handlers.base import SOLAR_TEMP_LOCAL_LOCATION
 from solar.core.handlers.base import TempFileHandler
 from solar.core.log import log
-from solar import errors
 
 
 class Shell(TempFileHandler):
@@ -36,15 +35,10 @@ def action(self, resource, action_name):
         self.transport_sync.copy(resource, self.dst, '/tmp')
         self.transport_sync.sync_all()
 
-        cmd = self.transport_run.run(
+        rst = self.transport_run.run(
             resource,
             'bash', action_file_name,
             use_sudo=True,
             warn_only=True
         )
-
-        if cmd.return_code:
-            raise errors.SolarError(
-                'Bash execution for {} failed with {}'.format(
-                    resource.name, cmd.return_code))
-        return cmd
+        self.verify_run_results(['bash', action_file_name], rst)

solar/core/resource/virtual_resource.py

@@ -266,14 +266,17 @@ def parse_list_input(r_input, args):
     connections = []
     assignments = {}
     for arg in args:
-        if is_connection(arg):
+        if isinstance(arg, dict):
+            n_connections, n_assign = parse_dict_input(
+                r_input, arg)
+            connections.extend(n_connections)
+            if n_assign:
+                add_assignment(assignments, r_input, n_assign)
+        elif is_connection(arg):
             c = parse_connection(r_input, arg)
             connections.append(c)
         else:
-            try:
-                assignments[r_input].append(arg)
-            except KeyError:
-                assignments[r_input] = [arg]
+            add_assignment(assignments, r_input, arg)
     return connections, assignments
 
 
@@ -293,6 +296,13 @@ def parse_dict_input(r_input, args):
     return connections, assignments
 
 
+def add_assignment(assignments, r_input, arg):
+    try:
+        assignments[r_input].append(arg)
+    except KeyError:
+        assignments[r_input] = [arg]
+
+
 def is_connection(arg):
     if isinstance(arg, basestring) and '::' in arg:
         return True

solar/core/transports/base.py

@@ -12,6 +12,9 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+from solar.core.log import log
+from solar import errors
+
 
 class Executor(object):
 
@@ -38,7 +41,13 @@ def valid(self, value):
 
     def run(self, transport):
         if self.valid:
-            self._executor(transport)
+            result = self._executor(transport)
+            if isinstance(result, tuple) and len(result) == 3:
+                # TODO Include file information in result
+                rc, out, err = result
+                log.debug('RC %s OUT %s ERR %s', rc, out, err)
+                if rc:
+                    raise errors.SolarError(err)
 
 
 class SolarRunResultWrp(object):

solar/core/transports/bat.py

@@ -17,8 +17,8 @@
 from solar.core.transports.base import SolarTransport
 from solar.core.transports.base import SyncTransport
 from solar.core.transports.rsync import RsyncSyncTransport
-from solar.core.transports.ssh import SSHRunTransport
 from solar.core.transports.ssh import SSHSyncTransport
+from solar.core.transports.ssh_raw import RawSSHRunTransport
 try:
     from solar.core.transports.solar_agent_transport import SolarAgentRunTransport  # NOQA
     from solar.core.transports.solar_agent_transport import SolarAgentSyncTransport  # NOQA
@@ -42,7 +42,7 @@
 
 
 KNOWN_RUN_TRANSPORTS = {
-    'ssh': SSHRunTransport
+    'ssh': RawSSHRunTransport
 }
 
 

solar/core/transports/rsync.py

@@ -12,11 +12,10 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-from fabric import api as fabric_api
-
 from solar.core.log import log
 from solar.core.transports.base import Executor
 from solar.core.transports.base import SyncTransport
+from solar.utils import execute
 
 # XXX:
 # currently we don't support key verification or acceptation
@@ -55,9 +54,8 @@ def copy(self, resource, _from, _to, use_sudo=False):
                          _from=_from,
                          _to=_to)
 
-        rsync_executor = lambda transport: fabric_api.local(
-            rsync_cmd
-        )
+        rsync_executor = lambda transport: execute(
+            rsync_cmd, shell=True)
 
         log.debug("RSYNC CMD: %r" % rsync_cmd)