Merge remote-tracking branch 'origin' into consider_0_instances_stopped

debian/changelog

@@ -1,3 +1,17 @@
+paasta-tools (0.16.13-yelp1) lucid; urgency=low
+
+  * 0.16.13 tagged with 'make release'
+    Commit:
+
+ -- Evan Krall <krall@yelp.com>  Thu, 07 Jan 2016 14:46:00 -0800
+
+paasta-tools (0.16.12-yelp1) UNRELEASED; urgency=medium
+
+  * 0.16.12 tagged with 'make release'
+    Commit:
+
+ -- kwa <kwa@kwa-MacBookPro>  Tue, 05 Jan 2016 14:50:38 -0800
+
 paasta-tools (0.16.11-yelp1) lucid; urgency=low
 
   * 0.16.11 tagged with 'make release'

docs/source/about/paasta_principles.rst

@@ -14,6 +14,35 @@ for the exact contract an app must meet to run on PaaSTA is documented in the
 Principles
 ----------
 
+0. A Bespoke PaaS is not Necessarily a Bad Thing
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Building your own PaaS is a non-trivial undertaking. The decision to build your
+own, even if it makes heavy use of existing components (like PaaSTA), should
+not be taken lightly.
+
+On the plus side, building your own PaaS can give your organization more value
+than adopting an existing commercial or third-party PaaS. The value comes from
+the fact that you can build your PaaS to meet your specific needs.
+Additionally, much of the existing engineering effort towards your current
+infrastructure can often be re-used. For example if you have an existing
+service-discovery mechanism, you can build a PaaS that integrates tightly
+with it, as opposed to replacing it with a different service-discovery mechanism
+from a third-party.
+
+There are of course downsides to building your own PaaS. There is certainly
+engineering effort involved, but that must be weighed against the
+engineering effort involved in *not* building your own PaaS (integration,
+training, and migration costs). You also lose the support of the community,
+or in the case of a commercial PaaS, the support of a vendor. Problems with
+your own PaaS are not "googleable". No new employees will know how your PaaS
+works, you cannot hire existing experts on it.
+
+PaaSTA was built because we believed we could provide business value by
+building our own PaaS to seamlessly meld with Yelp's existing infrastructure. At
+the same time PaaSTA strives to avoid the downsides of building your own PaaS
+by heavily reusing existing open source components.
+
 1. **Declarative** is better than **imperative**
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 

general_itests/fake_soa_configs_validate/fake_invalid_service/chronos-test-cluster.yaml

@@ -0,0 +1,5 @@
+# Invalid with validation error: You must specify a "schedule" in your configuration
+---
+chronos_job:
+  cpus: .1
+  mem: 100

general_itests/fake_soa_configs_validate/fake_invalid_service/marathon-test-cluster.yaml

@@ -0,0 +1,7 @@
+---
+main:
+    cpus: .1
+    mem: 100
+    instances: 1
+    env:
+      FOO: BAR

general_itests/fake_soa_configs_validate/fake_valid_service/chronos-test-cluster.yaml

@@ -0,0 +1,5 @@
+---
+chronos_job:
+  cpus: .1
+  mem: 100
+  schedule: 'R/2015-08-14T10:00:00+00:00/PT10M'

general_itests/fake_soa_configs_validate/fake_valid_service/marathon-test-cluster.yaml

@@ -0,0 +1,7 @@
+---
+main:
+    cpus: .1
+    mem: 100
+    instances: 1
+    env:
+      FOO: BAR

general_itests/steps/validate_steps.py

@@ -0,0 +1,57 @@
+# Copyright 2015 Yelp Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from behave import given, when, then
+
+from paasta_tools.cli.utils import x_mark
+from paasta_tools.utils import _run
+
+
+@given(u'a "{service_type}" service')
+@given(u'an "{service_type}" service')
+def given_service(context, service_type):
+    context.service = 'fake_%s_service' % service_type
+    context.soa_dir = 'fake_soa_configs_validate'
+
+
+@when(u'we run paasta validate')
+def run_paasta_validate(context):
+    validate_cmd = ("cli.py validate "
+                    "--yelpsoa-config-root %s "
+                    "--service %s " % (context.soa_dir, context.service))
+    context.validate_return_code, context.validate_output = _run(command=validate_cmd)
+    context.validate_output = context.validate_output.decode('utf-8')
+
+
+@then(u'it should have a return code of "{code:d}"')
+def see_expected_return_code(context, code):
+    print context.validate_output
+    print context.validate_return_code
+    print
+    assert context.validate_return_code == code
+
+
+@then(u'everything should pass')
+def validate_status_all_pass(context):
+    assert not context.validate_output or x_mark().decode('utf-8') not in context.validate_output
+
+
+@then(u'it should report an error in the output')
+def validate_status_something_fail(context):
+    assert x_mark().decode('utf-8') in context.validate_output
+
+
+@then(u'the output should contain \'{output_string}\'')
+def output_contains(context, output_string):
+    assert output_string in context.validate_output

general_itests/validate.feature

@@ -0,0 +1,15 @@
+Feature: paasta validate can be used
+
+  Scenario: Running paasta validate against a valid service
+    Given a "valid" service
+    When we run paasta validate
+    Then it should have a return code of "0"
+     And everything should pass
+     And the output should contain 'has a valid instance'
+
+  Scenario: Running paasta validate against an invalid service
+    Given an "invalid" service
+    When we run paasta validate
+    Then it should have a return code of "1"
+     And it should report an error in the output
+     And the output should contain 'You must specify a "schedule" in your configuration'

paasta_tools/cli/cmds/check.py

@@ -19,19 +19,21 @@
 import re
 import urllib2
 
-from paasta_tools.marathon_tools import get_all_namespaces_for_service
-from paasta_tools.monitoring_tools import get_team
+from paasta_tools.cli.utils import figure_out_service_name
 from paasta_tools.cli.utils import get_file_contents
-from paasta_tools.cli.utils import guess_service_name
 from paasta_tools.cli.utils import is_file_in_dir
+from paasta_tools.cli.utils import lazy_choices_completer
+from paasta_tools.cli.utils import list_services
 from paasta_tools.cli.utils import NoSuchService
 from paasta_tools.cli.utils import PaastaCheckMessages
 from paasta_tools.cli.utils import success
 from paasta_tools.cli.utils import validate_service_name
 from paasta_tools.cli.utils import x_mark
+from paasta_tools.marathon_tools import get_all_namespaces_for_service
+from paasta_tools.monitoring_tools import get_team
 from paasta_tools.utils import DEPLOY_PIPELINE_NON_DEPLOY_STEPS
-from paasta_tools.utils import get_service_instance_list
 from paasta_tools.utils import get_git_url
+from paasta_tools.utils import get_service_instance_list
 from paasta_tools.utils import list_clusters
 from paasta_tools.utils import PaastaColors
 from paasta_tools.utils import _run
@@ -54,6 +56,10 @@ def add_subparser(subparsers):
         description=help_text,
         help=help_text,
     )
+    check_parser.add_argument(
+        '-s', '--service',
+        help='The name of the service you wish to inspect. Defaults to autodetect.'
+    ).completer = lazy_choices_completer(list_services)
     check_parser.set_defaults(command=paasta_check)
 
 
@@ -287,7 +293,7 @@ def smartstack_check(service, service_path):
 def paasta_check(args):
     """Analyze the service in the PWD to determine if it is paasta ready
     :param args: argparse.Namespace obj created from sys.args by cli"""
-    service = guess_service_name()
+    service = figure_out_service_name(args)
     service_path = os.path.join('/nail/etc/services', service)
 
     service_dir_check(service)

paasta_tools/cli/cmds/local_run.py

@@ -412,7 +412,8 @@ def run_docker_container(
     command,
     healthcheck,
     healthcheck_only,
-    instance_config
+    instance_config,
+    soa_dir=service_configuration_lib.DEFAULT_SOA_DIR,
 ):
     """docker-py has issues running a container with a TTY attached, so for
     consistency we execute 'docker run' directly in both interactive and
@@ -450,7 +451,8 @@ def run_docker_container(
     )
     # http://stackoverflow.com/questions/4748344/whats-the-reverse-of-shlex-split
     joined_docker_run_cmd = ' '.join(pipes.quote(word) for word in docker_run_cmd)
-    healthcheck_mode, healthcheck_data = get_healthcheck_for_instance(service, instance, instance_config, random_port)
+    healthcheck_mode, healthcheck_data = get_healthcheck_for_instance(
+        service, instance, instance_config, random_port, soa_dir=soa_dir)
 
     sys.stdout.write('Running docker command:\n%s\n' % PaastaColors.grey(joined_docker_run_cmd))
     if interactive:
@@ -593,14 +595,15 @@ def configure_and_run_docker_container(docker_client, docker_hash, service, inst
     run_docker_container(
         docker_client=docker_client,
         service=service,
-        instance=args.instance,
+        instance=instance,
         docker_hash=docker_hash,
         volumes=volumes,
         interactive=args.interactive,
         command=command,
         healthcheck=args.healthcheck,
         healthcheck_only=args.healthcheck_only,
         instance_config=instance_config,
+        soa_dir=args.yelpsoa_config_root,
     )
 
 

paasta_tools/cli/cmds/validate.py

@@ -16,18 +16,23 @@
 import json
 import os
 import pkgutil
+import sys
 import yaml
 
 from glob import glob
 from jsonschema import Draft4Validator
 from jsonschema import FormatChecker
 from jsonschema import ValidationError
+
+from paasta_tools.chronos_tools import load_chronos_job_config
 from paasta_tools.cli.utils import failure
 from paasta_tools.cli.utils import get_file_contents
 from paasta_tools.cli.utils import lazy_choices_completer
 from paasta_tools.cli.utils import list_services
 from paasta_tools.cli.utils import PaastaColors
 from paasta_tools.cli.utils import success
+from paasta_tools.utils import list_all_instances_for_service
+from paasta_tools.utils import list_clusters
 
 
 SCHEMA_VALID = success("Successfully validated schema")
@@ -50,6 +55,17 @@
                   % (PaastaColors.cyan('SERVICE'), PaastaColors.cyan('-s'))
 
 
+def invalid_chronos_instance(cluster, instance, output):
+    return failure(
+        'chronos-%s.yaml has an invalid instance: %s.\n  %s\n  '
+        'More info:' % (cluster, instance, output),
+        "http://paasta.readthedocs.org/en/latest/yelpsoa_configs.html#chronos-clustername-yaml")
+
+
+def valid_chronos_instance(cluster, instance):
+    return success('chronos-%s.yaml has a valid instance: %s.' % (cluster, instance))
+
+
 def get_schema(file_type):
     """Get the correct schema to use for validation
 
@@ -80,7 +96,7 @@ def validate_schema(file_path, file_type):
         config_file = get_file_contents(file_path)
     except IOError:
         print '%s: %s' % (FAILED_READING_FILE, file_path)
-        return
+        return 1
     if extension == '.yaml':
         config_file_object = yaml.load(config_file)
     elif extension == '.json':
@@ -92,8 +108,10 @@ def validate_schema(file_path, file_type):
     except ValidationError as e:
         print '%s: %s' % (SCHEMA_INVALID, file_path)
         print '  Validation Message: %s' % e.message
+        return 1
     else:
-        print '%s: %s' % (SCHEMA_VALID, file_path)
+        print '%s: %s' % (SCHEMA_VALID, basename)
+        return 0
 
 
 def validate_all_schemas(service_path):
@@ -109,9 +127,13 @@ def validate_all_schemas(service_path):
         if os.path.islink(file_name):
             continue
         basename = os.path.basename(file_name)
+        returncode = 0
         for file_type in ['chronos', 'marathon']:
             if basename.startswith(file_type):
-                validate_schema(file_name, file_type)
+                tmp_returncode = validate_schema(file_name, file_type)
+                if tmp_returncode != 0:
+                    returncode = tmp_returncode
+        return returncode
 
 
 def add_subparser(subparsers):
@@ -140,13 +162,51 @@ def get_service_path(service, soa_dir):
     :param args: argparse.Namespace obj created from sys.args by cli
     """
     if service:
-        return os.path.join(soa_dir, service)
+        service_path = os.path.join(soa_dir, service)
     else:
         if soa_dir == os.getcwd():
-            return os.getcwd()
+            service_path = os.getcwd()
         else:
             print UNKNOWN_SERVICE
             return None
+    if not os.path.isdir(service_path):
+        print failure("%s is not a directory" % service_path,
+                      "http://paasta.readthedocs.org/en/latest/yelpsoa_configs.html")
+        return None
+    if not glob(os.path.join(service_path, "*.yaml")):
+        print failure("%s does not contain any .yaml files" % service_path,
+                      "http://paasta.readthedocs.org/en/latest/yelpsoa_configs.html")
+        return None
+    return service_path
+
+
+def path_to_soa_dir_service(service_path):
+    soa_dir = os.path.dirname(service_path)
+    service = os.path.basename(service_path)
+    return soa_dir, service
+
+
+def validate_chronos(service_path):
+    soa_dir, service = path_to_soa_dir_service(service_path)
+    instance_type = 'chronos'
+
+    returncode = 0
+    for cluster in list_clusters(service, soa_dir, instance_type):
+        for instance in list_all_instances_for_service(
+                service=service, clusters=[cluster], instance_type=instance_type,
+                soa_dir=soa_dir):
+            cjc = load_chronos_job_config(service, instance, cluster, False, soa_dir)
+            checks_passed, check_msgs = cjc.validate()
+
+            # Remove duplicate check_msgs
+            unique_check_msgs = list(set(check_msgs))
+
+            if not checks_passed:
+                print invalid_chronos_instance(cluster, instance, "\n  ".join(unique_check_msgs))
+                returncode = 1
+            else:
+                print valid_chronos_instance(cluster, instance)
+    return returncode
 
 
 def paasta_validate(args):
@@ -160,6 +220,17 @@ def paasta_validate(args):
     service_path = get_service_path(service, soa_dir)
 
     if service_path is None:
-        return 1
+        sys.exit(1)
+
+    returncode = 0
+
+    tmp_returncode = validate_all_schemas(service_path)
+    if tmp_returncode != 0:
+        returncode = tmp_returncode
+
+    tmp_returncode = validate_chronos(service_path)
+    if tmp_returncode != 0:
+        returncode = tmp_returncode
 
-    validate_all_schemas(service_path)
+    if returncode != 0:
+        sys.exit(returncode)

paasta_tools/cli/utils.py

@@ -305,7 +305,7 @@ def guess_instance(service, cluster, args):
     else:
         try:
             instances = list_all_instances_for_service(
-                service=service, cluster=cluster, instance_type=None, soa_dir=args.yelpsoa_config_root)
+                service=service, clusters=[cluster], instance_type=None, soa_dir=args.yelpsoa_config_root)
             if 'main' in instances:
                 instance = 'main'
             else: