From 97f769e469d2b503d7a71393533662a6a76f8443 Mon Sep 17 00:00:00 2001 From: John Carr Date: Mon, 25 May 2015 19:02:55 +0100 Subject: [PATCH 1/5] Remove the runner classes --- touchdown/__init__.py | 4 +- touchdown/aws/logs/__init__.py | 2 + touchdown/aws/logs/tail.py | 76 ++++++++-------- touchdown/core/__init__.py | 2 - touchdown/core/goals.py | 56 +++--------- touchdown/core/main.py | 29 +++--- touchdown/core/runner.py | 91 ------------------- touchdown/goals/__init__.py | 26 ++++++ touchdown/goals/action.py | 66 ++++++++++++++ touchdown/goals/apply.py | 34 +++++++ touchdown/goals/destroy.py | 31 +++++++ touchdown/goals/dot.py | 42 +++++++++ touchdown/goals/tail.py | 41 +++++++++ touchdown/tests/aws.py | 31 ++++--- .../tests/test_aws_cloudfront_distribution.py | 4 +- touchdown/tests/test_aws_route53_zone.py | 24 ++--- touchdown/tests/test_provisioner.py | 22 +++-- 17 files changed, 361 insertions(+), 220 deletions(-) delete mode 100644 touchdown/core/runner.py create mode 100644 touchdown/goals/__init__.py create mode 100644 touchdown/goals/action.py create mode 100644 touchdown/goals/apply.py create mode 100644 touchdown/goals/destroy.py create mode 100644 touchdown/goals/dot.py create mode 100644 touchdown/goals/tail.py diff --git a/touchdown/__init__.py b/touchdown/__init__.py index 6287a9954f..f83b556bcb 100644 --- a/touchdown/__init__.py +++ b/touchdown/__init__.py @@ -15,10 +15,10 @@ import touchdown.aws # noqa import touchdown.provisioner # noqa import touchdown.ssh # noqa +import touchdown.goals # noqa -from touchdown.core import Runner, Workspace +from touchdown.core import Workspace __all__ = [ - "Runner", "Workspace", ] diff --git a/touchdown/aws/logs/__init__.py b/touchdown/aws/logs/__init__.py index ecf80b1098..ca34fbf228 100644 --- a/touchdown/aws/logs/__init__.py +++ b/touchdown/aws/logs/__init__.py @@ -13,6 +13,8 @@ # limitations under the License. from .group import LogGroup +from . import tail # noqa + __all__ = [ 'LogGroup', diff --git a/touchdown/aws/logs/tail.py b/touchdown/aws/logs/tail.py index c7f6ec0aec..637eeba6ed 100644 --- a/touchdown/aws/logs/tail.py +++ b/touchdown/aws/logs/tail.py @@ -17,40 +17,44 @@ import time +from touchdown.core import plan from touchdown.core.datetime import parse_datetime_as_seconds - - -def tail(runner, log_group, start=None, end=None, follow=False): - plan = runner.goal.get_plan(log_group) - client = plan.client - - kwargs = { - 'logGroupName': log_group.name, - } - if start: - kwargs['startTime'] = parse_datetime_as_seconds(start) - if end: - kwargs['endTime'] = parse_datetime_as_seconds(end) - - def pull(kwargs, previous_events): - seen = set() - filters = {} - filters.update(kwargs) - results = client.filter_log_events(**filters) - while True: - for event in results.get('events', []): - seen.add(event['eventId']) - if event['eventId'] in previous_events: - continue - print(u"[{logStreamName}] {message}".format(**event)) - kwargs['startTime'] = event['timestamp'] - if 'nextToken' not in results: - break - filters['nextToken'] = results['nextToken'] - results = client.filter_log_events(**filters) - return seen - - seen = pull(kwargs, set()) - while follow: - seen = pull(kwargs, seen) - time.sleep(2) +from touchdown.aws.logs import LogGroup + + +class Plan(plan.Plan): + + name = "tail" + resource = LogGroup + + def tail(self, start, end, follow): + kwargs = { + 'logGroupName': self.resource.name, + } + if start: + kwargs['startTime'] = parse_datetime_as_seconds(start) + if end: + kwargs['endTime'] = parse_datetime_as_seconds(end) + + def pull(kwargs, previous_events): + seen = set() + filters = {} + filters.update(kwargs) + results = self.client.filter_log_events(**filters) + while True: + for event in results.get('events', []): + seen.add(event['eventId']) + if event['eventId'] in previous_events: + continue + print(u"[{logStreamName}] {message}".format(**event)) + kwargs['startTime'] = event['timestamp'] + if 'nextToken' not in results: + break + filters['nextToken'] = results['nextToken'] + results = self.client.filter_log_events(**filters) + return seen + + seen = pull(kwargs, set()) + while follow: + seen = pull(kwargs, seen) + time.sleep(2) diff --git a/touchdown/core/__init__.py b/touchdown/core/__init__.py index 6aa314d0d2..109d5a6e2f 100644 --- a/touchdown/core/__init__.py +++ b/touchdown/core/__init__.py @@ -12,11 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -from .runner import Runner from .workspace import Workspace __all__ = [ - "Runner", "Workspace", ] diff --git a/touchdown/core/goals.py b/touchdown/core/goals.py index e544f1f6cc..bebe1b4657 100644 --- a/touchdown/core/goals.py +++ b/touchdown/core/goals.py @@ -14,7 +14,7 @@ from __future__ import division -from . import dependencies, plan +from . import dependencies, plan, map, errors class GoalFactory(object): @@ -25,22 +25,23 @@ def __init__(self): def register(self, cls): self.goals[cls.name] = cls - def create(self, name, workspace, ui): - return self.goals[name](workspace, ui) + def create(self, name, workspace, ui, map=map.ParallelMap): + try: + goal_class = self.goals[name] + except KeyError: + raise errors.Error("No such goal '{}'".format(name)) + return goal_class(workspace, ui, map=map) class Goal(object): execute_in_reverse = False - def __init__(self, workspace, ui): + def __init__(self, workspace, ui, map=map.ParallelMap): self.ui = ui self.workspace = workspace self.resources = {} - self.reset_changes() - - def reset_changes(self): - self.changes = {} + self.Map = map def get_plan_order(self): return dependencies.DependencyMap(self.workspace, tips_first=False) @@ -56,14 +57,6 @@ def get_plan(self, resource): self.resources[resource] = plan return self.resources[resource] - def get_changes(self, resource): - if resource not in self.changes: - self.changes[resource] = list(self.get_plan(resource).get_actions()) - return self.changes[resource] - - def is_stale(self): - return len(self.changes) != 0 - def get_execution_order(self): return dependencies.DependencyMap(self.workspace, tips_first=self.execute_in_reverse) @@ -76,32 +69,7 @@ def get_plan_class(self, resource): return resource.meta.plans.get("describe", plan.NullPlan) -class Apply(Goal): - - name = "apply" - - def get_plan_class(self, resource): - if "destroy" in resource.policies: - return resource.meta.plans["destroy"] - - if "never-create" in resource.policies: - return resource.meta.plans["describe"] - - return resource.meta.plans.get("apply", resource.meta.plans.get("describe", plan.NullPlan)) - - -class Destroy(Goal): - - name = "destroy" - execute_in_reverse = True - - def get_plan_class(self, resource): - if "never-destroy" not in resource.policies: - return resource.meta.plans.get("destroy", resource.meta.plans.get("describe", plan.NullPlan)) - return resource.meta.plans.get("describe", plan.NullPlan) - - goals = GoalFactory() -goals.register(Describe) -goals.register(Apply) -goals.register(Destroy) +register = goals.register +create = goals.create +register(Describe) diff --git a/touchdown/core/main.py b/touchdown/core/main.py index 563f6ba50f..94e6533fba 100644 --- a/touchdown/core/main.py +++ b/touchdown/core/main.py @@ -16,9 +16,8 @@ import click -from touchdown.core.runner import ThreadedRunner, Runner from touchdown.core.workspace import Workspace -from touchdown.core import errors +from touchdown.core import errors, goals, map class ConsoleInterface(object): @@ -54,10 +53,13 @@ def confirm_plan(self, plan): return click.confirm("Do you want to continue?") -def get_runner(ctx, target): - if ctx.parent.params['serial']: - return Runner(target, ctx.parent.obj, ConsoleInterface()) - return ThreadedRunner(target, ctx.parent.obj, ConsoleInterface()) +def get_goal(ctx, target): + return goals.create( + target, + ctx.parent.obj, + ConsoleInterface(), + map=map.ParallelMap if not ctx.parent.params['serial'] else map.SerialMap + ) @click.group() @@ -77,7 +79,7 @@ def main(ctx, debug, serial): @main.command() @click.pass_context def apply(ctx): - r = get_runner(ctx, "apply") + r = get_goal(ctx, "apply") try: r.apply() except errors.Error as e: @@ -87,7 +89,7 @@ def apply(ctx): @main.command() @click.pass_context def destroy(ctx): - r = get_runner(ctx, "destroy") + r = get_goal(ctx, "destroy") try: r.apply() except errors.Error as e: @@ -97,15 +99,20 @@ def destroy(ctx): @main.command() @click.pass_context def plan(ctx): - r = get_runner(ctx, "apply") + r = get_goal(ctx, "apply") r.ui.render_plan(r.plan()) @main.command() @click.pass_context def dot(ctx): - r = get_runner(ctx, "apply") - click.echo(r.dot()) + get_goal(ctx, "dot").execute() + + +@main.command() +@click.pass_context +def tail(ctx): + get_goal(ctx, "tail").execute() if __name__ == "__main__": diff --git a/touchdown/core/runner.py b/touchdown/core/runner.py deleted file mode 100644 index 164f57606f..0000000000 --- a/touchdown/core/runner.py +++ /dev/null @@ -1,91 +0,0 @@ -# Copyright 2015 Isotoma Limited -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import division - -import logging - -from . import errors, goals, map - - -logger = logging.getLogger(__name__) - - -class Runner(object): - - Map = map.SerialMap - - def __init__(self, goal, workspace, ui): - try: - self.goal = goals.goals.create(goal, workspace, ui) - except KeyError: - raise errors.Error("No such goal '{}'".format(goal)) - - self.workspace = workspace - self.ui = ui - self.resources = {} - - def echo(self, text, **kwargs): - self.ui.echo(text, **kwargs) - - def dot(self): - graph = ["digraph ast {"] - - for node, deps in self.goal.get_plan_order().items(): - if not node.dot_ignore: - graph.append('{} [label="{}"];'.format(id(node), node)) - for dep in deps: - if not dep.dot_ignore: - graph.append("{} -> {};".format(id(node), id(dep))) - - graph.append("}") - return "\n".join(graph) - - def plan(self): - self.goal.reset_changes() - for progress in self.Map(self.goal.get_plan_order(), lambda e, r: self.goal.get_changes(r), self.echo): - self.echo("\r[{: >6.2%}] Building plan...".format(progress), nl=False) - self.echo("") - - for resource in self.goal.get_execution_order().all(): - changes = self.goal.get_changes(resource) - if changes: - yield resource, changes - - def apply_resource(self, echo, resource): - for change in self.goal.get_changes(resource): - description = list(change.description) - echo("[{}] {}".format(resource, description[0])) - for line in description[1:]: - echo("[{}] {}".format(resource, line)) - change.run() - - def apply_resources(self): - self.Map(self.goal.get_execution_order(), self.apply_resource, self.echo)() - - def apply(self): - plan = list(self.plan()) - - if not len(plan): - raise errors.NothingChanged("Planning stage found no changes were required.") - - if not self.ui.confirm_plan(plan): - return - - self.apply_resources() - - -class ThreadedRunner(Runner): - - Map = map.ParallelMap diff --git a/touchdown/goals/__init__.py b/touchdown/goals/__init__.py new file mode 100644 index 0000000000..d92d32252e --- /dev/null +++ b/touchdown/goals/__init__.py @@ -0,0 +1,26 @@ +# Copyright 2015 Isotoma Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .apply import Apply +from .destroy import Destroy +from .dot import Dot +from .tail import Tail + + +__all__ = [ + "Apply", + "Destroy", + "Dot", + "Tail", +] diff --git a/touchdown/goals/action.py b/touchdown/goals/action.py new file mode 100644 index 0000000000..8d06fb75b0 --- /dev/null +++ b/touchdown/goals/action.py @@ -0,0 +1,66 @@ +# Copyright 2015 Isotoma Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from touchdown.core import errors + + +class ActionGoalMixin(object): + + def __init__(self, *args, **kwargs): + super(ActionGoalMixin, self).__init__(*args, **kwargs) + self.reset_changes() + + def reset_changes(self): + self.changes = {} + + def get_changes(self, resource): + if resource not in self.changes: + self.changes[resource] = list(self.get_plan(resource).get_actions()) + return self.changes[resource] + + def plan(self): + self.reset_changes() + for progress in self.Map(self.get_plan_order(), lambda e, r: self.get_changes(r), self.ui.echo): + self.ui.echo("\r[{: >6.2%}] Building plan...".format(progress), nl=False) + self.ui.echo("") + + for resource in self.get_execution_order().all(): + changes = self.get_changes(resource) + if changes: + yield resource, changes + + def apply_resource(self, echo, resource): + for change in self.get_changes(resource): + description = list(change.description) + echo("[{}] {}".format(resource, description[0])) + for line in description[1:]: + echo("[{}] {}".format(resource, line)) + change.run() + + def apply_resources(self): + self.Map(self.get_execution_order(), self.apply_resource, self.ui.echo)() + + def is_stale(self): + return len(self.changes) != 0 + + def execute(self): + plan = list(self.plan()) + + if not len(plan): + raise errors.NothingChanged("Planning stage found no changes were required.") + + if not self.ui.confirm_plan(plan): + return + + self.apply_resources() diff --git a/touchdown/goals/apply.py b/touchdown/goals/apply.py new file mode 100644 index 0000000000..b06d3a137f --- /dev/null +++ b/touchdown/goals/apply.py @@ -0,0 +1,34 @@ +# Copyright 2015 Isotoma Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from touchdown.core import plan +from touchdown.core.goals import Goal, register +from touchdown.goals.action import ActionGoalMixin + + +class Apply(ActionGoalMixin, Goal): + + name = "apply" + + def get_plan_class(self, resource): + if "destroy" in resource.policies: + return resource.meta.plans["destroy"] + + if "never-create" in resource.policies: + return resource.meta.plans["describe"] + + return resource.meta.plans.get("apply", resource.meta.plans.get("describe", plan.NullPlan)) + + +register(Apply) diff --git a/touchdown/goals/destroy.py b/touchdown/goals/destroy.py new file mode 100644 index 0000000000..ac8d03f140 --- /dev/null +++ b/touchdown/goals/destroy.py @@ -0,0 +1,31 @@ +# Copyright 2015 Isotoma Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from touchdown.core import plan +from touchdown.core.goals import Goal, register +from touchdown.goals.action import ActionGoalMixin + + +class Destroy(ActionGoalMixin, Goal): + + name = "destroy" + execute_in_reverse = True + + def get_plan_class(self, resource): + if "never-destroy" not in resource.policies: + return resource.meta.plans.get("destroy", resource.meta.plans.get("describe", plan.NullPlan)) + return resource.meta.plans.get("describe", plan.NullPlan) + + +register(Destroy) diff --git a/touchdown/goals/dot.py b/touchdown/goals/dot.py new file mode 100644 index 0000000000..d795a6a8a3 --- /dev/null +++ b/touchdown/goals/dot.py @@ -0,0 +1,42 @@ +# Copyright 2015 Isotoma Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from touchdown.core import plan +from touchdown.core.goals import Goal, register + + +class Dot(Goal): + + name = "dot" + + def get_plan_class(self, resource): + return plan.NullPlan + + def get_digraph(self): + graph = ["digraph ast {"] + + for node, deps in self.get_plan_order().items(): + if not node.dot_ignore: + graph.append('{} [label="{}"];'.format(id(node), node)) + for dep in deps: + if not dep.dot_ignore: + graph.append("{} -> {};".format(id(node), id(dep))) + + graph.append("}") + return "\n".join(graph) + + def execute(self): + self.ui.echo(self.get_digraph()) + +register(Dot) diff --git a/touchdown/goals/tail.py b/touchdown/goals/tail.py new file mode 100644 index 0000000000..78b9ff006b --- /dev/null +++ b/touchdown/goals/tail.py @@ -0,0 +1,41 @@ +# Copyright 2015 Isotoma Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from touchdown.core import plan +from touchdown.core.goals import Goal, register + + +class Tail(Goal): + + name = "tail" + + def get_plan_class(self, resource): + return resource.meta.plans.get("tail", plan.NullPlan) + + def execute(self): + tailers = {} + + def _(e, r): + plan = self.get_plan(r) + if not plan.name: + return + tailers[plan.name] = plan + + for progress in self.Map(self.get_plan_order(), _, self.ui.echo): + self.ui.echo("\r[{: >6.2%}] Building plan...".format(progress), nl=False) + self.ui.echo("") + + print tailers + +register(Tail) diff --git a/touchdown/tests/aws.py b/touchdown/tests/aws.py index 04055a47db..ccc47764e6 100644 --- a/touchdown/tests/aws.py +++ b/touchdown/tests/aws.py @@ -24,9 +24,9 @@ import vcr -from touchdown.core import workspace, errors -from touchdown.core.runner import Runner +from touchdown.core import workspace, errors, goals from touchdown.core.main import ConsoleInterface +from touchdown.core.map import SerialMap from touchdown.core.utils import force_bytes from botocore.vendored.requests.exceptions import ConnectionError @@ -148,7 +148,12 @@ def __init__(self, *args, **kwargs): self.workspace = workspace.Workspace() self.aws = self.workspace.add_aws(access_key_id='dummy', secret_access_key='dummy', region='eu-west-1') - self.runner = Runner("apply", self.workspace, ConsoleInterface(interactive=False)) + self.goal = goals.create( + "apply", + self.workspace, + ConsoleInterface(interactive=False), + map=SerialMap + ) def tearDown(self): self._patcher.stop() @@ -165,7 +170,7 @@ def setUp(self): self.fixture_404 = "aws_{}_describe_404".format(self.resource.resource_name) self.fixture_create = "aws_{}_create".format(self.resource.resource_name) - self.plan = self.runner.goal.get_plan(self.resource) + self.plan = self.goal.get_plan(self.resource) self.base_url = 'https://{}.eu-west-1.amazonaws.com/'.format(self.plan.service_name) def setUpResource(self): @@ -173,16 +178,14 @@ def setUpResource(self): def test_no_change(self): self.responses.add_fixture("POST", self.base_url, self.fixture_found, expires=1) - self.runner.dot() - self.assertRaises(errors.NothingChanged, self.runner.apply) + self.assertRaises(errors.NothingChanged, self.goal.execute) self.assertEqual(self.plan.resource_id, self.expected_resource_id) def test_create(self): self.responses.add_fixture("POST", self.base_url, self.fixture_404, expires=1) self.responses.add_fixture("POST", self.base_url, self.fixture_create, expires=1) self.responses.add_fixture("POST", self.base_url, self.fixture_found) - self.runner.dot() - self.runner.apply() + self.goal.execute() self.assertEqual(self.plan.resource_id, self.expected_resource_id) @@ -226,11 +229,11 @@ def tearDown(self): self.stack.close() def apply(self): - self.apply_runner = Runner("apply", self.workspace, ConsoleInterface(interactive=False)) - self.apply_runner.apply() - self.assertRaises(errors.NothingChanged, self.apply_runner.apply) + self.apply_runner = goals.create("apply", self.workspace, ConsoleInterface(interactive=False), map=SerialMap) + self.apply_runner.execute() + self.assertRaises(errors.NothingChanged, self.apply_runner.execute) def destroy(self): - self.destroy_runner = Runner("destroy", self.workspace, ConsoleInterface(interactive=False)) - self.destroy_runner.apply() - self.assertRaises(errors.NothingChanged, self.destroy_runner.apply) + self.destroy_runner = goals.create("destroy", self.workspace, ConsoleInterface(interactive=False), map=SerialMap) + self.destroy_runner.execute() + self.assertRaises(errors.NothingChanged, self.destroy_runner.execute) diff --git a/touchdown/tests/test_aws_cloudfront_distribution.py b/touchdown/tests/test_aws_cloudfront_distribution.py index 4ac20e2b5f..f59295811a 100644 --- a/touchdown/tests/test_aws_cloudfront_distribution.py +++ b/touchdown/tests/test_aws_cloudfront_distribution.py @@ -110,7 +110,7 @@ def test_no_change(self): "aws_distribution_get", expires=1 ) - self.assertRaises(errors.NothingChanged, self.runner.apply) + self.assertRaises(errors.NothingChanged, self.goal.execute) self.assertEqual(self.plan.resource_id, self.expected_resource_id) # FIXME: Refactor tests so matching can be done in a generic way @@ -119,5 +119,5 @@ def test_create(self): self.responses.add_fixture("POST", self.base_url, self.fixture_create, expires=1) self.responses.add_fixture("GET", "https://cloudfront.amazonaws.com/2014-11-06/distribution", self.fixture_found) self.responses.add_fixture("GET", "https://cloudfront.amazonaws.com/2014-11-06/distribution/EDFDVBD6EXAMPLE", "aws_distribution_get") - self.runner.apply() + self.goal.execute() self.assertEqual(self.plan.resource_id, self.expected_resource_id) diff --git a/touchdown/tests/test_aws_route53_zone.py b/touchdown/tests/test_aws_route53_zone.py index 8a9257429f..0345e9dec0 100644 --- a/touchdown/tests/test_aws_route53_zone.py +++ b/touchdown/tests/test_aws_route53_zone.py @@ -32,7 +32,7 @@ def test_no_change(self): self.resource = self.aws.add_hosted_zone( name='example.com', ) - self.plan = self.runner.goal.get_plan(self.resource) + self.plan = self.goal.get_plan(self.resource) self.responses.add_fixture( "GET", @@ -40,7 +40,7 @@ def test_no_change(self): "aws_hosted_zone_rrset_0", ) self.responses.add_fixture("GET", "https://route53.amazonaws.com/2013-04-01/hostedzone", self.fixture_found) - self.assertRaises(errors.NothingChanged, self.runner.apply) + self.assertRaises(errors.NothingChanged, self.goal.execute) self.assertEqual(self.plan.resource_id, self.expected_resource_id) def test_no_change_1(self): @@ -54,7 +54,7 @@ def test_no_change_1(self): "values": ['127.0.0.1'], }] ) - self.plan = self.runner.goal.get_plan(self.resource) + self.plan = self.goal.get_plan(self.resource) self.responses.add_fixture( "GET", @@ -67,7 +67,7 @@ def test_no_change_1(self): "aws_hosted_zone_rrset_1", ) self.responses.add_fixture("GET", "https://route53.amazonaws.com/2013-04-01/hostedzone", self.fixture_found) - self.assertRaises(errors.NothingChanged, self.runner.apply) + self.assertRaises(errors.NothingChanged, self.goal.execute) self.assertEqual(self.plan.resource_id, self.expected_resource_id) # FIXME: Refactor tests so matching can be done in a generic way @@ -76,7 +76,7 @@ def test_create(self): self.resource = self.aws.add_hosted_zone( name='example.com', ) - self.plan = self.runner.goal.get_plan(self.resource) + self.plan = self.goal.get_plan(self.resource) self.responses.add_fixture( "GET", @@ -86,7 +86,7 @@ def test_create(self): self.responses.add_fixture("GET", "https://route53.amazonaws.com/2013-04-01/hostedzone", self.fixture_404, expires=1) self.responses.add_fixture("POST", self.base_url, self.fixture_create, expires=1) self.responses.add_fixture("GET", "https://route53.amazonaws.com/2013-04-01/hostedzone", self.fixture_found) - self.runner.apply() + self.goal.execute() self.assertEqual(self.plan.resource_id, self.expected_resource_id) def test_add_rrset(self): @@ -100,7 +100,7 @@ def test_add_rrset(self): "values": ['127.0.0.1'], }] ) - self.plan = self.runner.goal.get_plan(self.resource) + self.plan = self.goal.get_plan(self.resource) self.responses.add_fixture( "GET", @@ -113,7 +113,7 @@ def test_add_rrset(self): "aws_hosted_zone_rrset_change", ) self.responses.add_fixture("GET", "https://route53.amazonaws.com/2013-04-01/hostedzone", self.fixture_found) - self.runner.apply() + self.goal.execute() self.assertEqual(self.plan.resource_id, self.expected_resource_id) def test_delete_rrset(self): @@ -121,7 +121,7 @@ def test_delete_rrset(self): self.resource = self.aws.add_hosted_zone( name='example.com', ) - self.plan = self.runner.goal.get_plan(self.resource) + self.plan = self.goal.get_plan(self.resource) self.responses.add_fixture( "GET", @@ -134,7 +134,7 @@ def test_delete_rrset(self): "aws_hosted_zone_rrset_change", ) self.responses.add_fixture("GET", "https://route53.amazonaws.com/2013-04-01/hostedzone", self.fixture_found) - self.runner.apply() + self.goal.execute() self.assertEqual(self.plan.resource_id, self.expected_resource_id) def test_update_rrset(self): @@ -148,7 +148,7 @@ def test_update_rrset(self): "values": ['192.168.0.1'], }] ) - self.plan = self.runner.goal.get_plan(self.resource) + self.plan = self.goal.get_plan(self.resource) self.responses.add_fixture( "GET", @@ -161,5 +161,5 @@ def test_update_rrset(self): "aws_hosted_zone_rrset_change", ) self.responses.add_fixture("GET", "https://route53.amazonaws.com/2013-04-01/hostedzone", self.fixture_found) - self.runner.apply() + self.goal.execute() self.assertEqual(self.plan.resource_id, self.expected_resource_id) diff --git a/touchdown/tests/test_provisioner.py b/touchdown/tests/test_provisioner.py index 12ed320d45..147e45344e 100644 --- a/touchdown/tests/test_provisioner.py +++ b/touchdown/tests/test_provisioner.py @@ -16,9 +16,9 @@ import unittest import tempfile -from touchdown.core import workspace, errors, serializers -from touchdown.core.runner import Runner +from touchdown.core import workspace, errors, serializers, goals from touchdown.core.main import ConsoleInterface +from touchdown.core.map import SerialMap class TestCase(unittest.TestCase): @@ -27,12 +27,22 @@ def setUp(self): self.workspace = workspace.Workspace() def apply(self): - self.apply_runner = Runner("apply", self.workspace, ConsoleInterface(interactive=False)) - self.apply_runner.apply() + self.apply_runner = goals.create( + "apply", + self.workspace, + ConsoleInterface(interactive=False), + map=SerialMap + ) + self.apply_runner.execute() def test_destroy(self): - self.destroy_runner = Runner("destroy", self.workspace, ConsoleInterface(interactive=False)) - self.assertRaises(errors.NothingChanged, self.destroy_runner.apply) + self.destroy_runner = goals.create( + "destroy", + self.workspace, + ConsoleInterface(interactive=False), + map=SerialMap + ) + self.assertRaises(errors.NothingChanged, self.destroy_runner.execute) def test_file_apply(self): with tempfile.NamedTemporaryFile(delete=True) as fp: From a614912da42e463c2a1460c26c1dc3c8658cf452 Mon Sep 17 00:00:00 2001 From: John Carr Date: Mon, 25 May 2015 19:07:22 +0100 Subject: [PATCH 2/5] Remove stray print --- touchdown/goals/tail.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/touchdown/goals/tail.py b/touchdown/goals/tail.py index 78b9ff006b..6890d59d57 100644 --- a/touchdown/goals/tail.py +++ b/touchdown/goals/tail.py @@ -36,6 +36,4 @@ def _(e, r): self.ui.echo("\r[{: >6.2%}] Building plan...".format(progress), nl=False) self.ui.echo("") - print tailers - register(Tail) From 884e92b7b11fd8a2afd152879018f98ecf816bb3 Mon Sep 17 00:00:00 2001 From: John Carr Date: Mon, 25 May 2015 22:46:19 +0100 Subject: [PATCH 3/5] More refactoring --- touchdown/aws/account.py | 4 ++-- touchdown/aws/common.py | 31 +++++++++++++++++-------------- touchdown/aws/logs/tail.py | 4 +++- touchdown/core/plan.py | 5 ++++- touchdown/core/resource.py | 6 ++++++ touchdown/goals/apply.py | 7 +++---- touchdown/goals/destroy.py | 5 ++--- touchdown/goals/tail.py | 15 +++++++++------ 8 files changed, 46 insertions(+), 31 deletions(-) diff --git a/touchdown/aws/account.py b/touchdown/aws/account.py index 069c21ca9c..252e82e755 100644 --- a/touchdown/aws/account.py +++ b/touchdown/aws/account.py @@ -37,11 +37,11 @@ class Account(BaseAccount): root = argument.Resource(Workspace) -class Describe(Plan): +class Null(Plan): resource = Account default = True - name = "describe" + name = "null" _session = None @property diff --git a/touchdown/aws/common.py b/touchdown/aws/common.py index 2e49f268dd..8d108a905c 100644 --- a/touchdown/aws/common.py +++ b/touchdown/aws/common.py @@ -172,23 +172,10 @@ def run(self): ) -class SimpleDescribe(object): - - name = "describe" - - describe_filters = None - describe_notfound_exception = None - - signature = ( - Present('name'), - ) +class SimplePlan(object): _client = None - def __init__(self, runner, resource): - super(SimpleDescribe, self).__init__(runner, resource) - self.object = {} - @property def session(self): return self.parent.session @@ -200,6 +187,22 @@ def client(self): self._client = session.create_client(self.service_name) return self._client + +class SimpleDescribe(SimplePlan): + + name = "describe" + + describe_filters = None + describe_notfound_exception = None + + signature = ( + Present('name'), + ) + + def __init__(self, runner, resource): + super(SimpleDescribe, self).__init__(runner, resource) + self.object = {} + def get_describe_filters(self): return { self.key: self.resource.name diff --git a/touchdown/aws/logs/tail.py b/touchdown/aws/logs/tail.py index 637eeba6ed..892fe0603e 100644 --- a/touchdown/aws/logs/tail.py +++ b/touchdown/aws/logs/tail.py @@ -19,13 +19,15 @@ from touchdown.core import plan from touchdown.core.datetime import parse_datetime_as_seconds +from touchdown.aws import common from touchdown.aws.logs import LogGroup -class Plan(plan.Plan): +class Plan(common.SimplePlan, plan.Plan): name = "tail" resource = LogGroup + service_name = "logs" def tail(self, start, end, follow): kwargs = { diff --git a/touchdown/core/plan.py b/touchdown/core/plan.py index d7e753dfba..94e52fd23f 100644 --- a/touchdown/core/plan.py +++ b/touchdown/core/plan.py @@ -13,7 +13,7 @@ # limitations under the License. import six -from . import errors +from . import errors, resource class PlanType(type): @@ -76,6 +76,9 @@ def get_actions(self): class NullPlan(Plan): """ A plan that doesn't do anything """ + resource = resource.Resource + name = "null" + class ArgumentAssertion(object): diff --git a/touchdown/core/resource.py b/touchdown/core/resource.py index 1394393c8d..4a0023bdc4 100644 --- a/touchdown/core/resource.py +++ b/touchdown/core/resource.py @@ -77,6 +77,11 @@ def __init__(self): self.fields = {} self.field_order = [] + def get_plan(self, plan): + for cls in self.mro: + if hasattr(cls, "meta") and plan in cls.meta.plans: + return cls.meta.plans[plan] + def iter_fields_in_order(self): for name in self.field_order: yield self.fields[name] @@ -124,6 +129,7 @@ def __new__(meta_cls, class_name, bases, new_attrs): # Actually build a class cls = type.__new__(meta_cls, class_name, bases, new_attrs) + cls.meta.mro = cls.mro() # Allow fields to contribute to the class... for field in cls.meta.iter_fields_in_order(): diff --git a/touchdown/goals/apply.py b/touchdown/goals/apply.py index b06d3a137f..f7d4533064 100644 --- a/touchdown/goals/apply.py +++ b/touchdown/goals/apply.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from touchdown.core import plan from touchdown.core.goals import Goal, register from touchdown.goals.action import ActionGoalMixin @@ -23,12 +22,12 @@ class Apply(ActionGoalMixin, Goal): def get_plan_class(self, resource): if "destroy" in resource.policies: - return resource.meta.plans["destroy"] + return resource.meta.get_plan("destroy") if "never-create" in resource.policies: - return resource.meta.plans["describe"] + return resource.meta.get_plan("describe") - return resource.meta.plans.get("apply", resource.meta.plans.get("describe", plan.NullPlan)) + return resource.meta.get_plan("apply") or resource.meta.get_plan("describe") or resource.meta.get_plan("null") register(Apply) diff --git a/touchdown/goals/destroy.py b/touchdown/goals/destroy.py index ac8d03f140..51b1857fde 100644 --- a/touchdown/goals/destroy.py +++ b/touchdown/goals/destroy.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from touchdown.core import plan from touchdown.core.goals import Goal, register from touchdown.goals.action import ActionGoalMixin @@ -24,8 +23,8 @@ class Destroy(ActionGoalMixin, Goal): def get_plan_class(self, resource): if "never-destroy" not in resource.policies: - return resource.meta.plans.get("destroy", resource.meta.plans.get("describe", plan.NullPlan)) - return resource.meta.plans.get("describe", plan.NullPlan) + return resource.meta.get_plan("destroy") or resource.meta.get_plan("describe") or resource.meta.get_plan("null") + return resource.meta.get_plan("describe") or resource.meta.get_plan("null") register(Destroy) diff --git a/touchdown/goals/tail.py b/touchdown/goals/tail.py index 6890d59d57..8a3cbe3a04 100644 --- a/touchdown/goals/tail.py +++ b/touchdown/goals/tail.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from touchdown.core import plan from touchdown.core.goals import Goal, register @@ -21,19 +20,23 @@ class Tail(Goal): name = "tail" def get_plan_class(self, resource): - return resource.meta.plans.get("tail", plan.NullPlan) + plan_class = resource.meta.get_plan("tail") + if not plan_class: + plan_class = resource.meta.get_plan("null") + return plan_class def execute(self): tailers = {} def _(e, r): - plan = self.get_plan(r) - if not plan.name: - return - tailers[plan.name] = plan + p = self.get_plan(r) + if p.name == "tail": + tailers[p.resource.name] = p for progress in self.Map(self.get_plan_order(), _, self.ui.echo): self.ui.echo("\r[{: >6.2%}] Building plan...".format(progress), nl=False) self.ui.echo("") + tailers["application.log"].tail("5m", None, True) + register(Tail) From 4cf5b4c8d9512b58e25ff152b6a72f71405964bb Mon Sep 17 00:00:00 2001 From: John Carr Date: Tue, 26 May 2015 15:01:41 +0100 Subject: [PATCH 4/5] Get rid of click in favour of argparse --- requirements.txt | 1 - setup.py | 1 - touchdown/core/goals.py | 9 +++ touchdown/core/main.py | 120 ++++++++++++++++++------------------- touchdown/goals/action.py | 2 +- touchdown/goals/apply.py | 2 + touchdown/goals/destroy.py | 2 + touchdown/goals/dot.py | 4 +- touchdown/goals/tail.py | 17 +++++- 9 files changed, 91 insertions(+), 67 deletions(-) diff --git a/requirements.txt b/requirements.txt index 1bd6c4dc8e..de4784c64a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,5 @@ botocore==0.107.0 jmespath==0.7.1 -click==4.0 netaddr==0.7.14 contextlib2==0.4.0 paramiko==1.15.2 diff --git a/setup.py b/setup.py index d5177586f6..e1b3218ca0 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,6 @@ zip_safe=False, install_requires=[ 'six', - 'click', 'contextlib2', 'netaddr', 'fuselage>=0.0.6', diff --git a/touchdown/core/goals.py b/touchdown/core/goals.py index bebe1b4657..788b60b55a 100644 --- a/touchdown/core/goals.py +++ b/touchdown/core/goals.py @@ -25,6 +25,9 @@ def __init__(self): def register(self, cls): self.goals[cls.name] = cls + def registered(self): + return self.goals.items() + def create(self, name, workspace, ui, map=map.ParallelMap): try: goal_class = self.goals[name] @@ -43,6 +46,10 @@ def __init__(self, workspace, ui, map=map.ParallelMap): self.resources = {} self.Map = map + @classmethod + def setup_argparse(cls, parser): + pass + def get_plan_order(self): return dependencies.DependencyMap(self.workspace, tips_first=False) @@ -71,5 +78,7 @@ def get_plan_class(self, resource): goals = GoalFactory() register = goals.register +registered = goals.registered create = goals.create + register(Describe) diff --git a/touchdown/core/main.py b/touchdown/core/main.py index 94e6533fba..b636ee30a6 100644 --- a/touchdown/core/main.py +++ b/touchdown/core/main.py @@ -12,9 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging +from __future__ import print_function -import click +import argparse +import logging +import sys from touchdown.core.workspace import Workspace from touchdown.core import errors, goals, map @@ -27,92 +29,88 @@ def __init__(self, interactive=True): def echo(self, text, nl=True, **kwargs): if nl: - click.echo("{}\n".format(text), nl=False, **kwargs) + print("{}\n".format(text), end='') else: - click.echo("{}".format(text), nl=False, **kwargs) + print("{}".format(text), end='') + + def confirm(self, message): + response = raw_input('{} [Y/n] '.format(message)) + while response.lower() not in ('y', 'n', ''): + response = raw_input('{} [Y/n] '.format(message)) + return response.lower() == 'y' def render_plan(self, plan): for resource, actions in plan: - click.echo("%s:" % resource) + print("%s:" % resource) for action in actions: description = list(action.description) - click.echo(" * %s" % description[0]) + print(" * %s" % description[0]) for line in description[1:]: - click.echo(" %s" % line) - click.echo("") + print(" %s" % line) + print("") def confirm_plan(self, plan): - click.echo("Generated a plan to update infrastructure configuration:") - click.echo() + print("Generated a plan to update infrastructure configuration:") + print() self.render_plan(plan) if not self.interactive: return True - return click.confirm("Do you want to continue?") + return self.confirm("Do you want to continue?") -def get_goal(ctx, target): - return goals.create( - target, - ctx.parent.obj, - ConsoleInterface(), - map=map.ParallelMap if not ctx.parent.params['serial'] else map.SerialMap - ) +class SubCommand(object): + def __init__(self, goal, workspace, console): + self.goal = goal + self.workspace = workspace + self.console = console -@click.group() -@click.option('--debug/--no-debug', default=False, envvar='DEBUG') -@click.option('--serial/--parallel', default=False, envvar='SERIAL') -@click.pass_context -def main(ctx, debug, serial): - if debug: - logging.basicConfig(level=logging.DEBUG, format="%(name)s: %(message)s") - g = {"workspace": Workspace()} - with open("Touchdownfile") as f: - code = compile(f.read(), "Touchdownfile", "exec") - exec(code, g) - ctx.obj = g['workspace'] + def __call__(self, args): + try: + g = self.goal( + self.workspace, + self.console, + map.ParallelMap if not args.serial else map.SerialMap + ) + return g.execute(args) + except errors.Error as e: + self.console.echo(str(e)) + sys.exit(1) -@main.command() -@click.pass_context -def apply(ctx): - r = get_goal(ctx, "apply") - try: - r.apply() - except errors.Error as e: - raise click.ClickException(str(e)) +def configure_parser(parser, workspace, console): + parser.add_argument("--debug", default=False, action="store_true") + parser.add_argument("--serial", default=False, action="store_true") + sub = parser.add_subparsers() + for name, goal in goals.registered(): + p = sub.add_parser(name, help=getattr(goal, "__doc__", "")) + goal.setup_argparse(p) + p.set_defaults(func=SubCommand( + goal, + workspace, + console, + )) -@main.command() -@click.pass_context -def destroy(ctx): - r = get_goal(ctx, "destroy") - try: - r.apply() - except errors.Error as e: - raise click.ClickException(str(e)) - - -@main.command() -@click.pass_context -def plan(ctx): - r = get_goal(ctx, "apply") - r.ui.render_plan(r.plan()) +def main(): + g = {"workspace": Workspace()} + with open("Touchdownfile") as f: + code = compile(f.read(), "Touchdownfile", "exec") + exec(code, g) -@main.command() -@click.pass_context -def dot(ctx): - get_goal(ctx, "dot").execute() + parser = argparse.ArgumentParser(description="Manage your infrastructure") + console = ConsoleInterface() + configure_parser(parser, g['workspace'], console) + args = parser.parse_args() + if args.debug: + logging.basicConfig(level=logging.DEBUG, format="%(name)s: %(message)s") -@main.command() -@click.pass_context -def tail(ctx): - get_goal(ctx, "tail").execute() + args.func(args) if __name__ == "__main__": diff --git a/touchdown/goals/action.py b/touchdown/goals/action.py index 8d06fb75b0..e60b229b0a 100644 --- a/touchdown/goals/action.py +++ b/touchdown/goals/action.py @@ -54,7 +54,7 @@ def apply_resources(self): def is_stale(self): return len(self.changes) != 0 - def execute(self): + def execute(self, args): plan = list(self.plan()) if not len(plan): diff --git a/touchdown/goals/apply.py b/touchdown/goals/apply.py index f7d4533064..03eb098595 100644 --- a/touchdown/goals/apply.py +++ b/touchdown/goals/apply.py @@ -18,6 +18,8 @@ class Apply(ActionGoalMixin, Goal): + """ Converge infrastructure on the state defined """ + name = "apply" def get_plan_class(self, resource): diff --git a/touchdown/goals/destroy.py b/touchdown/goals/destroy.py index 51b1857fde..3bd73bdbab 100644 --- a/touchdown/goals/destroy.py +++ b/touchdown/goals/destroy.py @@ -18,6 +18,8 @@ class Destroy(ActionGoalMixin, Goal): + """ Tear down this infrastructure """ + name = "destroy" execute_in_reverse = True diff --git a/touchdown/goals/dot.py b/touchdown/goals/dot.py index d795a6a8a3..0049ac93b0 100644 --- a/touchdown/goals/dot.py +++ b/touchdown/goals/dot.py @@ -18,6 +18,8 @@ class Dot(Goal): + """ Generate a dot graph of all resources and their interconnections """ + name = "dot" def get_plan_class(self, resource): @@ -36,7 +38,7 @@ def get_digraph(self): graph.append("}") return "\n".join(graph) - def execute(self): + def execute(self, args): self.ui.echo(self.get_digraph()) register(Dot) diff --git a/touchdown/goals/tail.py b/touchdown/goals/tail.py index 8a3cbe3a04..3de227dd81 100644 --- a/touchdown/goals/tail.py +++ b/touchdown/goals/tail.py @@ -12,11 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +from touchdown.core import errors from touchdown.core.goals import Goal, register class Tail(Goal): + """ Inspect (and stream) your logs """ + name = "tail" def get_plan_class(self, resource): @@ -25,7 +28,14 @@ def get_plan_class(self, resource): plan_class = resource.meta.get_plan("null") return plan_class - def execute(self): + @classmethod + def setup_argparse(cls, parser): + parser.add_argument("stream", metavar="STREAM", type=str, help="The logstream to tail") + parser.add_argument("-f", "--follow", default=False, action="store_true", help="Don't exit and continue to print new events in the log stream") + parser.add_argument("-s", "--start", default="5m ago", action="store", help="The earliest event to retrieve") + parser.add_argument("-e", "--end", default=None, action="store", help="The latest event to retrieve") + + def execute(self, args): tailers = {} def _(e, r): @@ -37,6 +47,9 @@ def _(e, r): self.ui.echo("\r[{: >6.2%}] Building plan...".format(progress), nl=False) self.ui.echo("") - tailers["application.log"].tail("5m", None, True) + if args.stream not in tailers: + raise errors.Error("No such log group '{}'".format(args.stream)) + + tailers[args.stream].tail(args.start, args.end, args.tail) register(Tail) From e24d90d17f3b0078f7f605288bca35486b4523f1 Mon Sep 17 00:00:00 2001 From: John Carr Date: Tue, 26 May 2015 16:09:23 +0100 Subject: [PATCH 5/5] More refactoring. Add a 'rollback' goal. --- touchdown/aws/rds/__init__.py | 1 + touchdown/aws/rds/rollback.py | 189 ++++++++++++++++++++++++++++++++++ touchdown/core/main.py | 15 ++- touchdown/goals/__init__.py | 2 + touchdown/goals/action.py | 2 +- touchdown/goals/rollback.py | 52 ++++++++++ touchdown/goals/tail.py | 10 +- 7 files changed, 264 insertions(+), 7 deletions(-) create mode 100644 touchdown/aws/rds/rollback.py create mode 100644 touchdown/goals/rollback.py diff --git a/touchdown/aws/rds/__init__.py b/touchdown/aws/rds/__init__.py index 7a5fb04bfa..6d52134d93 100644 --- a/touchdown/aws/rds/__init__.py +++ b/touchdown/aws/rds/__init__.py @@ -16,6 +16,7 @@ from .database import Database from .point_in_time_restore import PointInTimeRestore from .snapshot_restore import SnapshotRestore +from . import rollback # noqa __all__ = [ diff --git a/touchdown/aws/rds/rollback.py b/touchdown/aws/rds/rollback.py new file mode 100644 index 0000000000..dd96dc23d6 --- /dev/null +++ b/touchdown/aws/rds/rollback.py @@ -0,0 +1,189 @@ +# Copyright 2015 Isotoma Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This code is not currently exposed publically. It is an example of how to +# stream from a aws log using the FilterLogEvents API. + +import time + +import jmespath +from botocore.exceptions import ClientError + +from touchdown.core import plan, errors +from touchdown.core.datetime import parse_datetime, now +from touchdown.aws import common +from touchdown.aws.rds import Database + + +def get_from_jmes(db, **kwargs): + new_kwargs = {} + for key, value in kwargs.items(): + if callable(value): + value = value() + if value: + newval = jmespath.search(value, db) + if newval: + new_kwargs[key] = newval + return new_kwargs + + +class Plan(common.SimplePlan, plan.Plan): + + name = "rollback" + resource = Database + service_name = "rds" + + def get_database(self, name): + try: + dbs = self.client.describe_db_instances(DBInstanceIdentifier=name).get('DBInstances', []) + except ClientError: + return None + return dbs[0] + + def validate(self, target, db_name, old_db_name): + db = self.get_database(db_name) + if not db: + raise errors.Error("Database {} not found?".format(db_name)) + + if self.get_database(old_db_name): + raise errors.Error("Database {} already exists - restore in progress?".format(old_db_name)) + + try: + datetime = parse_datetime(target) + if datetime > db['LatestRestorableTime']: + raise errors.Error("You cannot restore to {}. The most recent restorable time is {}".format( + datetime, + db['LatestRestorableTime'], + )) + if datetime < db['InstanceCreateTime']: + raise errors.Error('You cannot restore to {} because it is before the instance was created ({})'.format( + datetime, + db['InstanceCreateTime'], + )) + snapshots = self.client.describe_db_snapshots(DBInstanceIdentifier=db_name).get('DBSnapshots', []) + snapshots = filter(lambda snapshot: snapshot['InstanceCreateTime'] == db['InstanceCreateTime'], snapshots) + snapshots.sort(key=lambda snapshot: snapshot['SnapshotCreateTime']) + if not snapshots or datetime < snapshots[0]['SnapshotCreateTime']: + raise errors.Error('You cannot restore to {} because it is before the first available backup was created ({})'.format( + datetime, + snapshots[0]['SnapshotCreateTime'], + )) + + except ValueError: + try: + snapshots = self.client.describe_db_snapshots(DBInstanceIdentifier=db_name, DBSnapshotIdentifier=target).get('DBSnapshots', []) + except ClientError: + raise errors.Error("Could not find snapshot {}".format(target)) + if len(snapshots) == 0: + raise errors.Error("Could not find snapshot {}".format(target)) + return db + + def rollback(self, target): + db_name = self.resource.name + old_db_name = "{}-{:%Y%m%d%H%M%S}".format(db_name, now()) + + db = self.validate(target, db_name, old_db_name) + + print("Renaming {} to {}".format(db_name, old_db_name)) + self.client.modify_db_instance( + DBInstanceIdentifier=db_name, + NewDBInstanceIdentifier=old_db_name, + ApplyImmediately=True, + ) + + print("Waiting for rename to be completed") + while True: + try: + self.client.get_waiter("db_instance_available").wait( + DBInstanceIdentifier=old_db_name, + ) + except: + time.sleep(10) + else: + break + + kwargs = get_from_jmes( + db, + DBInstanceClass="DBInstanceClass", + Port="Endpoint.Port", + AvailabilityZone=lambda: "AvailabilityZone" if not db.get('MultiAZ', False) else None, + DBSubnetGroupName="DBSubnetGroup.DBSubnetGroupName", + MultiAZ="MultiAZ", + PubliclyAccessible="PubliclyAccessible", + AutoMinorVersionUpgrade="AutoMinorVersionUpgrade", + LicenseModel="LicenseModel", + DBName=lambda: "DBName" if db["Engine"] != 'postgres' else None, + Engine="Engine", + Iops="Iops", + OptionGroupName="OptionGroupMemberships[0].OptionGroupName", + StorageType="StorageType", + TdeCredentialArn="TdeCredentialArn", + ) + + print("Spinning database up from backup") + if target: + self.client.restore_db_instance_to_point_in_time( + SourceDBInstanceIdentifier=old_db_name, + TargetDBInstanceIdentifier=db_name, + RestoreTime=target, + **kwargs + ) + else: + self.client.restore_db_instance_from_db_snapshot( + DBInstanceIdentifier=db_name, + DBSnapshotIdentifier=target, + **kwargs + ) + + for i in range(10): + print("Waiting for database to be ready") + try: + self.client.get_waiter("db_instance_available").wait( + DBInstanceIdentifier=db_name, + ) + break + except Exception as e: + print(e) + time.sleep(10) + + kwargs = get_from_jmes( + db, + AllocatedStorage="AllocatedStorage", + DBSecurityGroups="DBSecurityGroups[?Status == 'active'].DBSecurityGroupName", + VpcSecurityGroupIds="VpcSecurityGroups[?Status == 'active'].VpcSecurityGroupId", + DBParameterGroupName="DBParameterGroups[0].DBParameterGroupName", + BackupRetentionPeriod="BackupRetentionPeriod", + PreferredBackupWindow="PreferredBackupWindow", + PreferredMaintenanceWindow="PreferredMaintenanceWindow", + EngineVersion="EngineVersion", + CACertificateIdentifier="CACertificateIdentifier", + ) + + print("Restoring database settings") + self.client.modify_db_instance( + DBInstanceIdentifier=db_name, + ApplyImmediately=True, + **kwargs + ) + + print("Waiting for database to be ready") + self.client.get_waiter("db_instance_available").wait( + DBInstanceIdentifier=db_name, + ) + + print("Deleting old database") + self.client.delete_db_instance( + DBInstanceIdentifier=old_db_name, + SkipFinalSnapshot=True, + ) diff --git a/touchdown/core/main.py b/touchdown/core/main.py index b636ee30a6..d29c472de8 100644 --- a/touchdown/core/main.py +++ b/touchdown/core/main.py @@ -15,6 +15,7 @@ from __future__ import print_function import argparse +import inspect import logging import sys @@ -68,6 +69,17 @@ def __init__(self, goal, workspace, console): self.workspace = workspace self.console = console + def get_args_and_kwargs(self, callable, namespace): + argspec = inspect.getargspec(callable) + args = [] + for arg in argspec.args[1:]: + args.append(getattr(namespace, arg)) + kwargs = {} + for k, v in namespace._get_kwargs(): + if k not in argspec.args and argspec.keywords: + kwargs[k] = v + return args, kwargs + def __call__(self, args): try: g = self.goal( @@ -75,7 +87,8 @@ def __call__(self, args): self.console, map.ParallelMap if not args.serial else map.SerialMap ) - return g.execute(args) + args, kwargs = self.get_args_and_kwargs(g.execute, args) + return g.execute(*args, **kwargs) except errors.Error as e: self.console.echo(str(e)) sys.exit(1) diff --git a/touchdown/goals/__init__.py b/touchdown/goals/__init__.py index d92d32252e..a69124d966 100644 --- a/touchdown/goals/__init__.py +++ b/touchdown/goals/__init__.py @@ -15,6 +15,7 @@ from .apply import Apply from .destroy import Destroy from .dot import Dot +from .rollback import Rollback from .tail import Tail @@ -22,5 +23,6 @@ "Apply", "Destroy", "Dot", + "Rollback", "Tail", ] diff --git a/touchdown/goals/action.py b/touchdown/goals/action.py index e60b229b0a..8d06fb75b0 100644 --- a/touchdown/goals/action.py +++ b/touchdown/goals/action.py @@ -54,7 +54,7 @@ def apply_resources(self): def is_stale(self): return len(self.changes) != 0 - def execute(self, args): + def execute(self): plan = list(self.plan()) if not len(plan): diff --git a/touchdown/goals/rollback.py b/touchdown/goals/rollback.py new file mode 100644 index 0000000000..fbc9b56c98 --- /dev/null +++ b/touchdown/goals/rollback.py @@ -0,0 +1,52 @@ +# Copyright 2015 Isotoma Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from touchdown.core import errors +from touchdown.core.goals import Goal, register + + +class Rollback(Goal): + + """ Rollback a database to a point in time or a backup """ + + name = "rollback" + + def get_plan_class(self, resource): + plan_class = resource.meta.get_plan("rollback") + if not plan_class: + plan_class = resource.meta.get_plan("null") + return plan_class + + @classmethod + def setup_argparse(cls, parser): + parser.add_argument("target", metavar="TARGET", type=str, help="A datetime or named snapshot to rollback to") + + def execute(self, target): + restorable = {} + + def _(e, r): + p = self.get_plan(r) + if p.name == "rollback": + restorable[p.resource.name] = p + + for progress in self.Map(self.get_plan_order(), _, self.ui.echo): + self.ui.echo("\r[{: >6.2%}] Building plan...".format(progress), nl=False) + self.ui.echo("") + + if "some_db" not in restorable: + raise errors.Error("No such resource '{}'".format("some_db")) + + restorable["some_db"].restore(target) + +register(Rollback) diff --git a/touchdown/goals/tail.py b/touchdown/goals/tail.py index 3de227dd81..ff09fa1e69 100644 --- a/touchdown/goals/tail.py +++ b/touchdown/goals/tail.py @@ -31,11 +31,11 @@ def get_plan_class(self, resource): @classmethod def setup_argparse(cls, parser): parser.add_argument("stream", metavar="STREAM", type=str, help="The logstream to tail") - parser.add_argument("-f", "--follow", default=False, action="store_true", help="Don't exit and continue to print new events in the log stream") + parser.add_argument("-f", "--follow", default=False, action="store_true", help="Don't exit and continue to print new events in the stream") parser.add_argument("-s", "--start", default="5m ago", action="store", help="The earliest event to retrieve") parser.add_argument("-e", "--end", default=None, action="store", help="The latest event to retrieve") - def execute(self, args): + def execute(self, stream, start="5m ago", end=None, follow=False): tailers = {} def _(e, r): @@ -47,9 +47,9 @@ def _(e, r): self.ui.echo("\r[{: >6.2%}] Building plan...".format(progress), nl=False) self.ui.echo("") - if args.stream not in tailers: - raise errors.Error("No such log group '{}'".format(args.stream)) + if stream not in tailers: + raise errors.Error("No such log group '{}'".format(stream)) - tailers[args.stream].tail(args.start, args.end, args.tail) + tailers[stream].tail(start, end, follow) register(Tail)