Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Initial commit

  • Loading branch information...
commit 8b84bea1b5c660ed2983e23b5eef9d800ef00bd6 0 parents
@priteau priteau authored
9 .gitignore
@@ -0,0 +1,9 @@
+*.egg
+*.egg-info
+*.pid
+*.pyc
+*.sock
+.coverage
+.idea
+build
+dist
194 LICENSE
@@ -0,0 +1,194 @@
+--------------------------------------------------------------------------------
+
+ Copyright 2012 University of Chicago
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+--------------------------------------------------------------------------------
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
0  README
No changes.
47 bin/ceictl
@@ -0,0 +1,47 @@
+#!/usr/bin/env python
+
+import argparse
+
+import yaml
+
+import ceiclient
+from ceiclient.client import SERVICES
+from ceiclient.connection import DashiCeiConnection
+
+parser = argparse.ArgumentParser(description='Client to control CEI services')
+
+parser.add_argument('--broker', '-b', action='store', dest='broker')
+parser.add_argument('--username', '-u', action='store', dest='username')
+parser.add_argument('--password', '-p', action='store', dest='password')
+parser.add_argument('--yaml', '-Y', action='store_const', const=True)
+
+subparsers = parser.add_subparsers(dest='service', help='Service to which to send a command')
+
+for service_name, service in SERVICES.items():
+ service_parser = subparsers.add_parser(service_name)
+ service_subparsers = service_parser.add_subparsers(dest='command', help='Command to send to the service')
+ for command_name, command in service.commands.items():
+ command(service_subparsers)
+
+opts = parser.parse_args()
+
+if __name__ == '__main__':
+ if opts.service not in SERVICES:
+ raise ValueError('Service %s is not supported' % opts.service)
+
+ service = SERVICES[opts.service]
+
+ if opts.command not in service.commands:
+ raise ValueError('Command %s is not supported by service %s' % (opts.command, opts.service))
+
+ command = service.commands[opts.command]
+ conn = DashiCeiConnection(opts.broker, opts.username, opts.password)
+ client = service(conn)
+
+ result = command.execute(client, opts)
+ if opts.yaml:
+ print(yaml.safe_dump(result, default_flow_style=False)),
+ else:
+ command.output(result)
+
+ conn.disconnect()
1  ceiclient/__init__.py
@@ -0,0 +1 @@
+import client
66 ceiclient/client.py
@@ -0,0 +1,66 @@
+import connection
+from commands import EPUMDescribe, EPUMList, EPUMReconfigure
+from commands import PDDispatch, PDDescribeProcess, PDDescribeProcesses, PDTerminateProcess, PDDump
+
+class CeiClient(object):
+
+ def __init__(self):
+ pass
+
+class EPUMClient(CeiClient):
+
+ dashi_name = 'epu_management_service'
+ name = 'epu'
+ help = 'Control the EPU Management Service'
+
+ def __init__(self, connection):
+ self._connection = connection
+
+ def describe_epu(self, name):
+ return self._connection.call(self.dashi_name, 'describe_epu', epu_name=name)
+
+ def list_epus(self):
+ return self._connection.call(self.dashi_name, 'list_epus')
+
+ def reconfigure_epu(self, name, config):
+ return self._connection.call(self.dashi_name, 'reconfigure_epu', epu_name=name, epu_config=config)
+
+ commands = {}
+ for command in [EPUMDescribe, EPUMList, EPUMReconfigure]:
+ commands[command.name] = command
+
+class PDClient(CeiClient):
+
+ dashi_name = 'processdispatcher'
+ name = 'process'
+ help = 'Control the Process Dispatcher Service'
+
+ def __init__(self, connection):
+ self._connection = connection
+
+ def dispatch_process(self, upid, spec, subscribers, constraints, immediate=False):
+ return self._connection.call(self.dashi_name, 'dispatch_process',
+ upid=upid, spec=spec,
+ subscribers=subscribers,
+ constraints=constraints,
+ immediate=immediate)
+
+ def describe_process(self, upid):
+ return self._connection.call(self.dashi_name, 'describe_process', upid=upid)
+
+ def describe_processes(self):
+ return self._connection.call(self.dashi_name, 'describe_processes')
+
+ def terminate_process(self, upid):
+ return self._connection.call(self.dashi_name, 'terminate_process', upid=upid)
+
+ def dump(self):
+ return self._connection.call(self.dashi_name, 'dump')
+
+ commands = {}
+ for command in [PDDispatch, PDDescribeProcess, PDDescribeProcesses, PDTerminateProcess, PDDump]:
+ commands[command.name] = command
+
+SERVICES = {}
+for service in [EPUMClient, PDClient]:
+ SERVICES[service.name] = service
161 ceiclient/commands.py
@@ -0,0 +1,161 @@
+import uuid
+
+from jinja2 import Template
+import yaml
+
+class CeiCommand(object):
+
+ def __init__(self, subparsers):
+ pass
+
+class EPUMDescribe(CeiCommand):
+
+ name = 'describe'
+ output_template = '''Name: {{ result.name }}
+Engine configuration: EPU worker type = {{ result.config.engine_conf.epuworker_type }}
+ Preserve N = {{ result.config.engine_conf.preserve_n }}
+General: Engine class = {{ result.config.general.engine_class }}
+Health: Monitor health = {{result.config.health.monitor_health}}'''
+
+ def __init__(self, subparsers):
+ parser = subparsers.add_parser(self.name)
+ parser.add_argument('epu_name', action='store', help='The EPU to describe')
+
+ @staticmethod
+ def execute(client, opts):
+ return client.describe_epu(opts.epu_name)
+
+ @staticmethod
+ def output(result):
+ template = Template(EPUMDescribe.output_template)
+ print template.render(result=result)
+
+class EPUMList(CeiCommand):
+
+ name = 'list'
+
+ def __init__(self, subparsers):
+ subparsers.add_parser(self.name)
+
+ @staticmethod
+ def execute(client, opts):
+ return client.list_epus()
+
+ @staticmethod
+ def output(result):
+ for epu_name in result:
+ print epu_name
+
+class EPUMReconfigure(CeiCommand):
+
+ name = 'reconfigure'
+
+ def __init__(self, subparsers):
+ parser = subparsers.add_parser(self.name)
+ parser.add_argument('epu_name', action='store', help='The EPU to reconfigure')
+ parser.add_argument('--bool', dest='updated_kv_bool', action='append', help='Key to modify in the EPU configuration with a boolean value')
+ parser.add_argument('--int', dest='updated_kv_int', action='append', help='Key to modify in the EPU configuration with a integer value')
+ parser.add_argument('--string', dest='updated_kv_string', action='append', help='Key to modify in the EPU configuration with a string value')
+
+ @staticmethod
+ def _TODO_hash_helper():
+ pass
+
+ @staticmethod
+ def execute(client, opts):
+ # TODO Create a hash for the merge
+ return client.reconfigure_epu(opts.epu_name, opts.updated_key_value)
+
+ @staticmethod
+ def output(result):
+ print(result)
+
+class PDDispatch(CeiCommand):
+
+ name = 'dispatch'
+ def __init__(self, subparsers):
+
+ parser = subparsers.add_parser(self.name)
+ parser.add_argument('process_spec', metavar='process_spec.yml')
+ parser.add_argument('--immediate', '-i', action='store_const', const=True, default=False)
+
+ @staticmethod
+ def execute(client, opts):
+ try:
+ with open(opts.process_spec) as f:
+ process_spec = yaml.load(f)
+ except Exception, e:
+ print "Problem reading process specification file %s: %s" % e
+ sys.exit(1)
+
+ return client.dispatch_process(str(uuid.uuid4().hex), process_spec, None, None, opts.immediate)
+
+ @staticmethod
+ def output(result):
+ print(result)
+
+class PDDescribeProcesses(CeiCommand):
+
+ name = 'list'
+
+ def __init__(self, subparsers):
+ parser = subparsers.add_parser(self.name)
+
+ @staticmethod
+ def execute(client, opts):
+ return client.describe_processes()
+
+ @staticmethod
+ def output(result):
+ print(result)
+
+class PDTerminateProcess(CeiCommand):
+
+ name = 'kill'
+
+ def __init__(self, subparsers):
+ parser = subparsers.add_parser(self.name)
+ parser.add_argument('process_id', action='store', help='The PID of the process to kill')
+
+ @staticmethod
+ def execute(client, opts):
+ return client.terminate_process(opts.process_id)
+
+ @staticmethod
+ def output(result):
+ print(result)
+
+class PDDescribeProcess(CeiCommand):
+
+ name = 'describe'
+
+ def __init__(self, subparsers):
+ parser = subparsers.add_parser(self.name)
+ parser.add_argument('process_id', action='store', help='The PID of the process to describe')
+
+ @staticmethod
+ def execute(client, opts):
+ return client.describe_process(opts.process_id)
+
+ @staticmethod
+ def output(result):
+ print(result)
+
+class PDDump(CeiCommand):
+
+ name = 'dump'
+
+ def __init__(self, subparsers):
+ parser = subparsers.add_parser(self.name)
+
+ @staticmethod
+ def execute(client, opts):
+ return client.dump()
+
+ @staticmethod
+ def output(result):
+ print(result)
+
+# TODO Other dashi calls for the PD:
+#dt_state
+#heartbeat, sender_kwarg='sender'
29 ceiclient/connection.py
@@ -0,0 +1,29 @@
+from dashi import DashiConnection
+from dashi.bootstrap import DEFAULT_EXCHANGE
+
+class CeiConnection(object):
+ """Abstract class defining the interface to talk with CEI services"""
+
+ def call(self, service, operation, **kwargs):
+ pass
+
+class DashiCeiConnection(CeiConnection):
+
+ _name = 'ceiclient'
+
+ def __init__(self, broker, username, password):
+ self.amqp_broker = broker
+ self.amqp_username = username
+ self.amqp_password = password
+ self.amqp_port = 5672
+
+ self.dashi_connection = DashiConnection(self._name,
+ 'amqp://%s:%s@%s:%s//' % (self.amqp_username,
+ self.amqp_password, self.amqp_broker,
+ self.amqp_port), DEFAULT_EXCHANGE)
+
+ def call(self, service, operation, timeout=5, **kwargs):
+ return self.dashi_connection.call(service, operation, timeout, **kwargs)
+
+ def disconnect(self):
+ self.dashi_connection.disconnect()
8 ceiclient/tests/test_client.py
@@ -0,0 +1,8 @@
+from mock import Mock, patch
+
+from ceiclient.client import EPUMClient
+
+def test_create_epum_client():
+ conn = Mock()
+ epum_client = EPUMClient(conn)
+ assert epum_client.dashi_name == 'epu_management_service'
67 ceiclient/tests/test_commands.py
@@ -0,0 +1,67 @@
+from mock import Mock, patch
+from nose.tools import raises
+import argparse
+import pprint
+
+from ceiclient.commands import EPUMDescribe, EPUMList, EPUMReconfigure
+from ceiclient.commands import PDDispatch
+
+class TestCommandParsing:
+
+ def setUp(self):
+ self.parser = argparse.ArgumentParser()
+ self.subparsers = self.parser.add_subparsers(dest='command')
+
+ def test_command_parsing_epum_describe_ok(self):
+ EPUMDescribe(self.subparsers)
+ opts = self.parser.parse_args(['describe', 'epu1'])
+ assert opts.command == 'describe'
+ assert opts.epu_name == 'epu1'
+
+ @raises(SystemExit)
+ def test_command_parsing_epum_describe_failing_wrong_command(self):
+ EPUMDescribe(self.subparsers)
+ opts = self.parser.parse_args(['list', 'epu1'])
+
+ @raises(SystemExit)
+ def test_command_parsing_epum_describe_failing_missing_argument(self):
+ EPUMDescribe(self.subparsers)
+ opts = self.parser.parse_args(['describe'])
+
+ def test_command_parsing_epum_list_ok(self):
+ EPUMList(self.subparsers)
+ opts = self.parser.parse_args(['list'])
+ assert opts.command == 'list'
+
+ @raises(SystemExit)
+ def test_command_parsing_epum_list_failing(self):
+ EPUMList(self.subparsers)
+ opts = self.parser.parse_args(['describe'])
+
+ def test_command_parsing_epum_reconfigure_ok(self):
+ EPUMReconfigure(self.subparsers)
+ boolkvpair = 'health.monitor_health=false'
+ intkvpair = 'engine_conf.preserve_n=42'
+ stringkvpair = 'general.engine_class=epu.decisionengine.impls.simplest.SimplestEngine'
+ opts = self.parser.parse_args(['reconfigure', 'epu1', '--bool', boolkvpair, '--int', intkvpair, '--string', stringkvpair])
+
+ assert opts.command == 'reconfigure'
+ assert opts.epu_name == 'epu1'
+ assert opts.updated_kv_bool == [boolkvpair]
+ assert opts.updated_kv_int == [intkvpair]
+ assert opts.updated_kv_string == [stringkvpair]
+
+ @raises(SystemExit)
+ def test_command_parsing_epum_reconfigure_failing_wrong_command(self):
+ EPUMReconfigure(self.subparsers)
+ opts = self.parser.parse_args(['describe', 'epu1', '42'])
+
+ @raises(SystemExit)
+ def test_command_parsing_epum_reconfigure_failing_not_enough_arguments(self):
+ EPUMReconfigure(self.subparsers)
+ opts = self.parser.parse_args(['describe', 'epu1'])
+
+ @raises(SystemExit)
+ def test_command_parsing_epum_reconfigure_failing_wrong_arguments(self):
+ EPUMReconfigure(self.subparsers)
+ opts = self.parser.parse_args(['describe', 'epu1', 'notanumber'])
14 ceiclient/tests/test_connection.py
@@ -0,0 +1,14 @@
+from mock import Mock, patch
+import dashi
+
+from ceiclient.connection import DashiCeiConnection
+
+def test_set_up_dashi_cei_connection():
+ server = 'localhost'
+ username = 'guest'
+ password = 'guest'
+
+ with patch('ceiclient.connection.DashiConnection') as mock:
+ dashi_cei_conn = DashiCeiConnection(server, username, password)
+ mock.assert_called_with('ceiclient', 'amqp://%s:%s@%s:5672//' %
+ (username, password, server), dashi.bootstrap.DEFAULT_EXCHANGE)
4 examples/process_spec.yml
@@ -0,0 +1,4 @@
+run_type: supd
+parameters:
+ argv: []
+ exec: "true"
4 setup.cfg
@@ -0,0 +1,4 @@
+[easy_install]
+#see http://peak.telecommunity.com/DevCenter/EasyInstall#configuration-files
+find_links = http://ooici.net/releases
+ http://github.com/nimbusproject/dashi/tarball/master#egg=dashi-0.1
17 setup.py
@@ -0,0 +1,17 @@
+#!/usr/bin/env python
+
+from distutils.core import setup
+from setuptools import setup, find_packages
+
+setup(name='ceiclient',
+ version='0.1',
+ description='Client tools for the OOI Common Execution Infrastructure',
+ license='Apache 2.0',
+ author='Nimbus team',
+ author_email='nimbus@mcs.anl.gov',
+ packages=['ceiclient'],
+ scripts=['bin/ceictl'],
+ install_requires=['dashi', 'Jinja2', 'PyYAML'],
+ setup_requires=['nose'],
+ tests_require=['coverage', 'mock', 'nose'],
+ test_suite='nose.collector')
Please sign in to comment.
Something went wrong with that request. Please try again.